Accept the insignificant commas and trailing dots that stock files use

src/JbeamEdit/Parsing/Jbeam.hs

@@ -18,7 +18,6 @@ import Data.Char (isSpace)
 import Data.Functor (($>))
 import Data.Maybe (isJust, isNothing)
 import Data.Monoid.Extra (mwhen)
-import Data.Scientific (Scientific)
 import Data.Text (Text)
 import Data.Text qualified as T
 import Data.Text.Encoding (decodeUtf8Lenient)
@@ -36,7 +35,7 @@ import JbeamEdit.Parsing.Common
 import Text.Megaparsec ((<?>), (<|>))
 import Text.Megaparsec qualified as MP
 import Text.Megaparsec.Byte qualified as B
-import Text.Megaparsec.Byte.Lexer qualified as L (decimal, scientific)
+import Text.Megaparsec.Byte.Lexer qualified as L (scientific)
 import Text.Megaparsec.Char qualified as C
 
 data ParseState = ParseState
@@ -51,7 +50,7 @@ separatorParser :: JbeamParser ()
 separatorParser = do
   ws1 <- MP.takeWhileP Nothing wordIsSpace
   comma <- MP.optional (MP.label "comma" $ byteChar ',')
-  ws2 <- MP.takeWhileP Nothing wordIsSpace
+  ws2 <- MP.takeWhileP Nothing (\x -> wordIsSpace x || toChar x == ',')
 
   let nl = toWord8 '\n'
       ws1Newlines = LBS.count nl ws1
@@ -84,14 +83,12 @@ numberParser = do
       else pure ""
   let signFactor = if char == toWord8 '-' then negate else id
   before <- MP.getInput
-  value <- MP.try intAsScientific <|> L.scientific
+  value <- L.scientific <?> "decimal number or integer"
   after <- MP.getInput
+  _ <- MP.optional (MP.try (byteChar '.' <* MP.notFollowedBy B.digitChar))
   let rawBytes = LBS.take (LBS.length before - LBS.length after) before
       rawText = decodeUtf8Lenient (LBS.toStrict rawBytes)
   pure $ Number (mkNumberValue (signText <> rawText) (signFactor value))
-  where
-    intAsScientific :: JbeamParser Scientific
-    intAsScientific = fromIntegral <$> intDecimalParser
 
 associationDirection :: ParseState -> AssociationDirection
 associationDirection st = bool PreviousNode NextNode (lastNodeEndedWithNewline st)
@@ -151,10 +148,6 @@ stringParser = parseWord8s String string
     emptyString = C.string "\"\"" >> pure []
     string = emptyString <|> validString
 
-intDecimalParser :: JbeamParser Integer
-intDecimalParser =
-  L.decimal <* MP.notFollowedBy (byteChar '.' <|> byteChar 'e' <|> byteChar 'E')
-
 scalarParser :: JbeamParser Node
 scalarParser =
   tryScalarParsers
@@ -203,7 +196,10 @@ objectKeyParser = do
   _ <- skipWhiteSpace
   key <- MP.try (stringParser <?> "string")
   _ <- skipWhiteSpace
+  _ <- MP.optional (byteChar ',')
+  _ <- skipWhiteSpace
   _ <- byteChar ':'
+  _ <- MP.optional (byteChar ',')
   value <- nodeParser
   pure $ ObjectKey (key, value)
 
@@ -215,7 +211,12 @@ objectParser = do
   pure . Object $ ObjectValue (V.fromList elems)
 
 topNodeParser :: JbeamParser Node
-topNodeParser = nodeParser <* skipWhiteSpace <* MP.eof
+topNodeParser =
+  nodeParser
+    <* skipWhiteSpace
+    <* MP.optional (byteChar ',')
+    <* skipWhiteSpace
+    <* MP.eof
 
 parseNodesState
   :: JbeamParser a

test/Parsing/JbeamSpec.hs

@@ -4,6 +4,7 @@ module Parsing.JbeamSpec (
 
 import Data.ByteString.Lazy (ByteString)
 import Data.ByteString.Lazy qualified as BS (readFile)
+import Data.Either (isRight)
 import Data.Vector (fromList)
 import Data.Void (Void)
 import JbeamEdit.Parsing.Common.Helpers
@@ -186,13 +187,57 @@ invalidSpec =
   where
     expLabels = foldMap elabel ["a valid scalar", "object", "array"]
 
+{- | Shapes that BeamNG reads and our grammar rejects, from issue #230. A comma
+is insignificant to the game, closer to white space than to structure, and a
+number may end on its decimal point.
+
+Each spec says only that the input is accepted, because what the formatter
+writes back for each shape is not decided yet.
+-}
+acceptsFragment :: (String, String) -> Spec
+acceptsFragment (desc, input) =
+  describe desc . it "is accepted, because the game reads it" $
+    parseNodesState' nodeParser input `shouldSatisfy` isRight
+
+acceptsDocument :: (String, String) -> Spec
+acceptsDocument (desc, input) =
+  describe desc . it "is accepted, because the game reads it" $
+    parseNodes (textToLazyByteString input) `shouldSatisfy` isRight
+
+gameReadableSpecs :: [Spec]
+gameReadableSpecs =
+  map
+    acceptsFragment
+    [
+      ( "an array with a stray comma where an element would be"
+      , "[\"id\", ,\"idRef:\"]"
+      )
+    , ("a comma after the colon of a key", "{\"innerfender_R\":, {\"a\": 1}}")
+    , ("a comma before the colon of a key", "{\"spoke1\",: {\"a\": 1}}")
+    , ("a number ending on its decimal point", "[\"f1\", 0., -1.66]")
+    ]
+    ++ [acceptsDocument ("a comma after the root object", "{\"a\": 1},\n")]
+
 invalidNumberSpec :: Spec
 invalidNumberSpec =
   describe
     "should fail parsing Number when there is space after the negative sign"
     . works
     $ parseNodesState numberParser "- 0.3"
-      `shouldFailWith` err 1 (utok (toWord8 ' ') <> elabel "digit" <> elabel "integer")
+      `shouldFailWith` err 1 (utok (toWord8 ' ') <> elabel "decimal number or integer")
+
+numberWithBadDecimalPoint :: Spec
+numberWithBadDecimalPoint =
+  describe
+    "should consume and discard a trailing period with no decimal digits"
+    . works
+    $ case parseNodesState ((,) <$> numberParser <*> MP.getInput) "3." of
+      Left bundle ->
+        expectationFailure $
+          "expected successful parse, got error:\n" <> MP.errorBundlePretty bundle
+      Right (node, remaining) -> do
+        node `shouldBe` Number (mkNumberValue "3" 3)
+        remaining `shouldBe` ""
 
 topNodeSpec :: FilePath -> FilePath -> Spec
 topNodeSpec inFilename outFilename = do
@@ -238,8 +283,10 @@ spec :: Spec
 spec = do
   mapM_ (applyParserSpec nodeParser) specs
   invalidSpec
+  sequence_ gameReadableSpecs
   invalidNumberSpec
   invalidTopNodeSpec
+  numberWithBadDecimalPoint
   topNodeSpecs
   where
     specs =