best-effort-json-parser/src/parse.spec.ts

Parse incomplete json text in best-effort manner

import sinon from 'sinon'
import { expect } from 'chai'
import { parse, setErrorLogger } from './parse'

let onExtraTokenSpy: sinon.SinonSpy
let originalOnExtraToken: typeof parse.onExtraToken
let muteLog = true
beforeEach(() => {
  if (muteLog) {
    onExtraTokenSpy = sinon.fake()
    originalOnExtraToken = parse.onExtraToken
    parse.onExtraToken = onExtraTokenSpy
  } else {
    onExtraTokenSpy = sinon.spy(parse, 'onExtraToken')
  }
})
afterEach(() => {
  if (muteLog) {
    parse.onExtraToken = originalOnExtraToken
  } else {
    sinon.restore()
  }
})

describe('parser TestSuit', function () {
  context('number', () => {
    it('should parse positive integer', function () {
      expect(parse(`42`)).equals(42)
    })
    it('should parse negative integer', function () {
      expect(parse(`-42`)).equals(-42)
    })

    it('should parse positive float', function () {
      expect(parse(`12.34`)).equals(12.34)
    })
    it('should parse negative float', function () {
      expect(parse(`-12.34`)).equals(-12.34)
    })

    it('should parse incomplete positive float', function () {
      expect(parse(`12.`)).equals(12)
    })
    it('should parse incomplete negative float', function () {
      expect(parse(`-12.`)).equals(-12)
    })
    it('should parse incomplete negative integer', function () {
      expect(parse(`-`)).equals(-0)
    })

    it('should preserve invalid number', function () {
      expect(parse(`1.2.3.4`)).equals('1.2.3.4')
    })
  })

  context('string', () => {
    it('should parse string', function () {
      expect(parse(`"I am text"`)).equals('I am text')
      expect(parse(`"I'm text"`)).equals("I'm text")
      expect(parse(`"I\\"m text"`)).equals('I"m text')
    })
    it('should parse incomplete string', function () {
      expect(parse(`"I am text`)).equals('I am text')
      expect(parse(`"I'm text`)).equals("I'm text")
      expect(parse(`"I\\"m text`)).equals('I"m text')
    })
  })

  context('boolean', () => {
    it('should parse boolean', function () {
      expect(parse(`true`)).equals(true)
      expect(parse(`false`)).equals(false)
    })

    function testIncomplete(str: string, val: boolean) {
      for (let i = str.length; i >= 1; i--) {
        expect(parse(str.slice(0, i))).equals(val)
      }
    }

    it('should parse incomplete true', function () {
      testIncomplete(`true`, true)
    })
    it('should parse incomplete false', function () {
      testIncomplete(`false`, false)
    })
  })

  context('array', () => {
    it('should parse empty array', function () {
      expect(parse(`[]`)).deep.equals([])
    })
    it('should parse number array', function () {
      expect(parse(`[1,2,3]`)).deep.equals([1, 2, 3])
    })
    it('should parse incomplete array', function () {
      expect(parse(`[1,2,3`)).deep.equals([1, 2, 3])
      expect(parse(`[1,2,`)).deep.equals([1, 2])
      expect(parse(`[1,2`)).deep.equals([1, 2])
      expect(parse(`[1,`)).deep.equals([1])
      expect(parse(`[1`)).deep.equals([1])
      expect(parse(`[`)).deep.equals([])
    })
  })

  context('object', () => {
    it('should parse simple object', function () {
      let o = { a: 'apple', b: 'banana' }
      expect(parse(JSON.stringify(o))).deep.equals(o)
      expect(parse(JSON.stringify(o, null, 2))).deep.equals(o)
      expect(parse(`{"a":"apple","b":"banana"}`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{"a": "apple","b": "banana"}`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{"a": "apple", "b": "banana"}`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{"a" : "apple", "b" : "banana"}`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{ "a" : "apple", "b" : "banana" }`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{ "a" : "apple" , "b" : "banana" }`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
    })
    it('should parse incomplete simple object', function () {
      expect(parse(`{"a":"apple","b":"banana"`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{"a":"apple","b":"banana`)).deep.equals({
        a: 'apple',
        b: 'banana',
      })
      expect(parse(`{"a":"apple","b":"b`)).deep.equals({ a: 'apple', b: 'b' })
      expect(parse(`{"a":"apple","b":"`)).deep.equals({ a: 'apple', b: '' })
      expect(parse(`{"a":"apple","b":`)).deep.equals({
        a: 'apple',
        b: undefined,
      })
      expect(parse(`{"a":"apple","b"`)).deep.equals({
        a: 'apple',
        b: undefined,
      })
      expect(parse(`{"a":"apple","b`)).deep.equals({ a: 'apple', b: undefined })
      expect(parse(`{"a":"apple","`)).deep.equals({
        'a': 'apple',
        '': undefined,
      })
      expect(parse(`{"a":"apple",`)).deep.equals({ a: 'apple' })
      expect(parse(`{"a":"apple"`)).deep.equals({ a: 'apple' })
      expect(parse(`{"a":"apple`)).deep.equals({ a: 'apple' })
      expect(parse(`{"a":"a`)).deep.equals({ a: 'a' })
      expect(parse(`{"a":"`)).deep.equals({ a: '' })
      expect(parse(`{"a":`)).deep.equals({ a: undefined })
      expect(parse(`{"a"`)).deep.equals({ a: undefined })
      expect(parse(`{"a`)).deep.equals({ a: undefined })
      expect(parse(`{"`)).deep.equals({ '': undefined })
      expect(parse(`{`)).deep.equals({})
    })
  })

  context('complex object', () => {
    it('should parse complete complex object', function () {
      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float": 12.34
        }
      }
}`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: 12.34,
          },
        },
      })
    })
    it('should parse incomplete complex object', function () {
      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float": 12.34
        }
      }`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: 12.34,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float": 12.34
        }`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: 12.34,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float": 12.34`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: 12.34,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float": 12.`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: 12,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,
          "float":`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
            float: undefined,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {
          "int": 42,`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {
            int: 42,
          },
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj": {`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: {},
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
        "obj":`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
          obj: undefined,
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], { "int": 42, "flo`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], { int: 42, flo: undefined }],
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 12.34], {`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 12.34], {}],
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [42, 12.34, [42, 1`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [42, 12.34, [42, 1]],
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float": 12.34,
        "arr": [`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: 12.34,
          arr: [],
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "float": 12.34 }],
      "obj": {
        "int": 42,
        "float"`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, float: 12.34 }],
        obj: {
          int: 42,
          float: undefined,
        },
      })

      expect(
        parse(`{
      "int": 42,
      "float": 12.34,
      "arr": [42, 12.34, [42, 12.34], { "int": 42, "flo`),
      ).deep.equals({
        int: 42,
        float: 12.34,
        arr: [42, 12.34, [42, 12.34], { int: 42, flo: undefined }],
      })
    })
  })

  context('invalid inputs', () => {
    it('should throw error on invalid (not incomplete) json text', function () {
      // spy the error logger
      let spy = sinon.fake()
      setErrorLogger(spy)

      expect(() => parse(`:atom`)).to.throws()
      expect(spy.called).be.true
      expect(spy.firstCall.firstArg).is.string('no parser registered for ":"')

      // restore the error logger
      setErrorLogger(console.error)
    })
    it('should complaint on extra tokens', function () {
      expect(parse(`[1] 2`)).deep.equals([1])
      expect(onExtraTokenSpy.called).be.true
      expect(parse.lastParseReminding).equals(' 2')
    })
  })

  context('extra space', () => {
    it('should parse complete json with extra space', function () {
      expect(parse(` [1] `)).deep.equals([1])
    })
    it('should parse incomplete json with extra space', function () {
      expect(parse(` [1 `)).deep.equals([1])
    })
  })

  context('invalid but understandable json', () => {
    context('string newline', () => {
      it('should parse escaped newline', function () {
        expect(parse(`"line1\\nline2"`)).equals('line1\nline2')
        expect(
          parse(/* javascript */ `
            {
              "essay": "global health.\n\nDuring my tenure ..."
            }
          `),
        ).deep.equals({
          essay: 'global health.\n\nDuring my tenure ...',
        })
      })
      it('should parse non-escaped newline', function () {
        expect(parse(`"line1\nline2"`)).equals('line1\nline2')
      })
      it('should parse non-escaped newline inside string value', function () {
        expect(parse(`{"key":"line1\\nline2`)).deep.equals({
          key: 'line1\nline2',
        })
        expect(parse(`{"key":"line1\nline2`)).deep.equals({
          key: 'line1\nline2',
        })
        expect(parse(`{"key":"line1\n`)).deep.equals({ key: 'line1\n' })
        expect(parse(`{\n\t"key":"line1\n`)).deep.equals({ key: 'line1\n' })
        expect(parse(`{\n\t"key":"line1\nline2"\n}`)).deep.equals({
          key: 'line1\nline2',
        })
      })
    })
    context('string non-escaped characters', function () {
      it('should parse \\t', function () {
        expect(parse(`"text\t"`)).equals(`text\t`)
        expect(parse(`"text\\t"`)).equals(`text\t`)
      })
      it('should parse \\r', function () {
        expect(parse(`"text\r"`)).equals(`text\r`)
        expect(parse(`"text\\r"`)).equals(`text\r`)
      })
    })
    context('string quote', () => {
      it('should parse string with double quote', function () {
        expect(parse(`"str"`)).equals('str')
      })
      it('should parse string with single quote', function () {
        expect(parse(`'str'`)).equals('str')
      })
      it('should parse single-quoted string with double quotes in payload', function () {
        expect(parse(`'{"refresh_token":"xxxx"}'`)).equals(
          '{"refresh_token":"xxxx"}',
        )
      })
      it('should parse string without double quote', function () {
        expect(parse(`str`)).equals('str')
      })
      it('should parse array of string without double quote', function () {
        expect(parse(`[a,b,c]`)).deep.equals(['a', 'b', 'c'])
      })
      it('should parse string with backticks', function () {
        expect(parse(`\`"Alice's"\``)).equals(`"Alice's"`)
        expect(
          parse(`[
            \`double quote: "\`,
            \`single quote: '\`,
            ${'`backtick: \\``'}
          ]`),
        ).deep.equals([`double quote: "`, `single quote: '`, 'backtick: `'])
      })
    })
    context('object key', () => {
      it('should parse object key with double quote', function () {
        expect(parse(`{ "int" : 42 }`)).deep.equals({ int: 42 })
      })
      it('should parse object key with single quote', function () {
        expect(parse(`{ 'int' : 42 }`)).deep.equals({ int: 42 })
      })
      it('should parse object key without double quote', function () {
        expect(parse(`{ int : 42 }`)).deep.equals({ int: 42 })
      })
    })
  })

  context('falsy values', () => {
    it('should parse empty string', function () {
      expect(parse('')).equals('')
    })
    it('should parse undefined', function () {
      expect(parse(undefined)).to.be.undefined
    })
    it('should parse null', function () {
      expect(parse(null)).to.be.null
    })
  })

  context('incomplete escaped characters', function () {
    it('should ignore an incomplete escaped character (such as control character \\n)', function () {
      expect(parse(`"the newline\n`)).equals('the newline\n')
      expect(parse(`"the newline\\n`)).equals('the newline\n')
      expect(parse(`"the newline\\`)).equals('the newline')
      expect(parse(`"the newline\n\\`)).equals('the newline\n')
      expect(parse(`"the newline\\n\\`)).equals('the newline\n')
      expect(parse(`"the newline\\\\`)).equals('the newline\\')
    })
    it('should ignore incomplete escape character in object value', function () {
      expect(parse('{"a":"\n"')).deep.equals({ a: '\n' })
      expect(parse('{"a":"\n')).deep.equals({ a: '\n' })
      expect(parse('{"a":"\\n"')).deep.equals({ a: '\n' })
      expect(parse('{"a":"\\')).deep.equals({ a: '' })
    })
  })

  context('comment in json', () => {
    it('should ignore inline comment', function () {
      // test with //
      let text = `{
        "a": 1, // comment
        "b": 2
      }`
      expect(parse(text)).deep.equals({ a: 1, b: 2 })
    })
    it('should ignore multi-line comment', function () {
      // test with /* */
      let text = `{
        "a": 1, /* line 1
        line 2
        line 3 */
        "b": 2
      }`
      expect(parse(text)).deep.equals({ a: 1, b: 2 })
    })
    it('should not strip comments inside strings', function () {
      let text = `{
        "comment": "// this is not a comment",
        "block": "/* neither is this */",
        "value": 42
      }`
      expect(parse(text)).deep.equals({
        comment: '// this is not a comment',
        block: '/* neither is this */',
        value: 42,
      })
    })
    it('should handle comments at beginning and end', function () {
      let text = `// start comment
      {
        "a": 1,
        "b": 2
      } // end comment`
      expect(parse(text)).deep.equals({ a: 1, b: 2 })
    })
    it('should handle mixed comment types', function () {
      let text = `{
        "a": 1, // inline comment
        /* multi-line
           comment */
        "b": 2
      }`
      expect(parse(text)).deep.equals({ a: 1, b: 2 })
    })
    it('should handle empty comments', function () {
      let text = `{
        "a": 1, //
        "b": 2, /**/
        "c": 3
      }`
      expect(parse(text)).deep.equals({ a: 1, b: 2, c: 3 })
    })
    it('should handle comments with special characters', function () {
      let text = `{
        "a": 1, // comment with "quotes" and 'single quotes'
        "b": 2 /* comment with { } [ ] */
      }`
      expect(parse(text)).deep.equals({ a: 1, b: 2 })
    })
    it('should handle html style of comments', function () {
      let text = `[
        "line 1", <!-- comment -->
        "line 2"
      ]`
      expect(parse(text)).deep.equals(['line 1', 'line 2'])
    })
  })
})