File: json0.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 from io import BufferedReader, BytesIO 27 from sys import argv, exit, stderr, stdin, stdout 28 29 30 info = ''' 31 json0 [filepath/URI...] 32 33 JSON-0 converts/fixes JSON/pseudo-JSON input into minimal JSON output. 34 35 Besides minimizing bytes, this tool also adapts almost-JSON input into valid 36 JSON, since it ignores comments and trailing commas, neither of which are 37 supported in JSON, but which are still commonly used. 38 39 It also turns single-quoted strings into proper double-quoted ones, as well 40 as change invalid 2-digit `\\x` hexadecimal escapes into JSON's 4-digit `\\u` 41 hexadecimal escapes. When backslashes in strings are followed by an invalid 42 escape letter, the backslash is ignored. 43 44 Output is always a single line of valid JSON, ending with a line-feed. 45 ''' 46 47 # handle standard help cmd-line options, quitting right away in that case 48 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'): 49 print(info.strip()) 50 exit(0) 51 52 53 # note: using regexes doesn't seem to speed-up number/string-handling 54 55 56 def read(r, size: int) -> bytes: 57 global pos, linenum 58 59 chunk = r.read(size) 60 if not chunk: 61 return chunk 62 63 if not (10 in chunk): 64 pos += len(chunk) 65 return chunk 66 67 for b in chunk: 68 if b == 10: 69 pos = 1 70 linenum += 1 71 else: 72 pos += 1 73 return chunk 74 75 76 def skip_byte(r) -> None: 77 global pos, linenum 78 79 chunk = r.read(1) 80 if not chunk: 81 return 82 83 if chunk[0] == 10: 84 pos = 1 85 linenum += 1 86 else: 87 pos += 1 88 89 90 def peek_byte(r) -> int: 91 chunk = r.peek(64) 92 if len(chunk) > 0: 93 return chunk[0] 94 return -1 95 96 97 def handle_array(w, r) -> None: 98 seek_next = seek_next_token 99 100 n = 0 101 lead = peek_byte(r) 102 end = 0 103 if lead < 0: 104 raise ValueError('unexpected end of input data, before "]"') 105 if lead == 91: # ord('[') 106 end = 93 # ord(']') 107 elif lead == 40: # ord('(') 108 end = 41 # ord(')') 109 else: 110 raise ValueError('expected "[" or "("') 111 skip_byte(r) 112 w.write(b'[') 113 114 while True: 115 # whitespace/comments may precede the next item/comma 116 seek_next(r) 117 b = peek_byte(r) 118 if b < 0: 119 raise ValueError('unexpected end of input data, before "]"') 120 121 comma = b == 44 # ord(',') 122 123 if comma: 124 skip_byte(r) 125 # whitespace/comments may follow the comma 126 seek_next(r) 127 b = peek_byte(r) 128 if b < 0: 129 raise ValueError('unexpected end of input data, before "]"') 130 131 if b == end: 132 skip_byte(r) 133 w.write(b']') 134 return 135 136 if n > 0: 137 if not comma: 138 raise ValueError('missing a comma between array values') 139 w.write(b',') 140 141 b = peek_byte(r) 142 if b > 0: 143 handlers[b](w, r) 144 n += 1 145 146 147 def handle_double_quoted_string(w, r) -> None: 148 skip_byte(r) 149 w.write(b'"') 150 handle_inner_string(w, r, 34) # ord('"') 151 w.write(b'"') 152 153 154 def handle_dot(w, r) -> None: 155 skip_byte(r) 156 # precede the leading decimal dot with a 0 157 w.write(b'0.') 158 159 # handle decimals, which in this case aren't optional, as a leading 160 # dot is what led to this point 161 if copy_digits(w, r) < 1: 162 raise ValueError('expected numeric digits, but found none') 163 164 165 def handle_false(w, r) -> None: 166 demand(r, b'false') 167 w.write(b'false') 168 169 170 def handle_False(w, r) -> None: 171 demand(r, b'False') 172 w.write(b'false') 173 174 175 def handle_invalid(w, r) -> None: 176 b = peek_byte(r) 177 if b < 0: 178 raise ValueError('unexpected end of input data') 179 # raise ValueError(f'unexpected JSON byte-value {b}') 180 if 32 < b <= 126: 181 msg = f'unexpected symbol {chr(b)}' 182 else: 183 msg = f'unexpected byte-value {b}' 184 raise ValueError(msg) 185 186 187 def handle_negative(w, r) -> None: 188 skip_byte(r) 189 w.write(b'-') 190 191 if peek_byte(r) == 46: # ord('.') 192 skip_byte(r) 193 w.write(b'0.') 194 if copy_digits(w, r) < 1: 195 raise ValueError('expected numeric digits, but found none') 196 else: 197 handle_number(w, r) 198 199 200 def handle_null(w, r) -> None: 201 demand(r, b'null') 202 w.write(b'null') 203 204 205 def handle_None(w, r) -> None: 206 demand(r, b'None') 207 w.write(b'null') 208 209 210 def handle_number(w, r) -> None: 211 # handle integer part 212 if copy_digits(w, r) < 1: 213 raise ValueError('expected numeric digits, but found none') 214 # ignore optional trailing 'n', used in javascript bigint literals 215 if peek_byte(r) == 110: # ord('n') 216 skip_byte(r) 217 return 218 219 # handle optional decimals 220 b = peek_byte(r) 221 if b == 46: # ord('.') 222 skip_byte(r) 223 w.write(b'.') 224 if copy_digits(w, r) < 1: 225 # follow a trailing decimal dot with a 0 226 w.write(b'0') 227 228 # handle optional exponent 229 if b == 101 or b == 69: # ord('e'), ord('E') 230 skip_byte(r) 231 w.write(b'e' if b == 101 else b'E') 232 b = peek_byte(r) 233 if b == 43: # ord('+') 234 skip_byte(r) 235 elif b == 45: # ord('-') 236 w.write(b'-') 237 skip_byte(r) 238 if copy_digits(w, r) < 1: 239 raise ValueError('expected numeric digits, but found none') 240 241 242 def handle_object(w, r) -> None: 243 seek_next = seek_next_token 244 245 num_pairs = 0 246 skip_byte(r) 247 w.write(b'{') 248 249 while True: 250 # whitespace/comments may precede the next item/comma 251 seek_next(r) 252 b = peek_byte(r) 253 if b < 0: 254 raise ValueError('unexpected end of input data, before "}"') 255 256 comma = b == 44 # ord(',') 257 258 if comma: 259 skip_byte(r) 260 # whitespace/comments may follow the comma 261 seek_next(r) 262 b = peek_byte(r) 263 if b < 0: 264 raise ValueError('unexpected end of input data, before "}"') 265 266 if b == 125: # ord('}') 267 skip_byte(r) 268 w.write(b'}') 269 return 270 271 if num_pairs > 0: 272 if not comma: 273 raise ValueError('missing a comma between key-value pairs') 274 w.write(b',') 275 276 demand_string(w, r) 277 # whitespace/comments may follow the key 278 seek_next(r) 279 demand(r, b':') 280 w.write(b':') 281 # whitespace/comments may follow the colon 282 seek_next(r) 283 b = peek_byte(r) 284 if b > 0: 285 handlers[b](w, r) 286 num_pairs += 1 287 288 289 def handle_positive(w, r) -> None: 290 # do nothing with the leading plus sign, which isn't allowed in JSON 291 skip_byte(r) 292 293 if peek_byte(r) == 46: # ord('.') 294 skip_byte(r) 295 w.write(b'0.') 296 if copy_digits(w, r) < 1: 297 raise ValueError('expected numeric digits, but found none') 298 else: 299 handle_number(w, r) 300 301 302 def handle_single_quoted_string(w, r) -> None: 303 skip_byte(r) 304 w.write(b'"') 305 handle_inner_string(w, r, 39) # ord('\'') 306 w.write(b'"') 307 308 309 def demand_string(w, r) -> None: 310 quote = peek_byte(r) 311 if quote < 0: 312 msg = 'unexpected end of input, instead of a string quote' 313 raise ValueError(msg) 314 315 if quote == 34: # ord('"') 316 handle_double_quoted_string(w, r) 317 return 318 319 if quote == 39: # ord('\'') 320 handle_single_quoted_string(w, r) 321 return 322 323 if 32 < quote <= 126: # ord(' '), ord('~') 324 msg = f'expected ", or even \', but got "{chr(quote)}" instead' 325 else: 326 msg = f'expected ", or even \', but got byte "{quote}" instead' 327 raise ValueError(msg) 328 329 330 def handle_inner_string(w, r, quote: int) -> None: 331 esc = False 332 bad_hex_msg = 'invalid hexadecimal symbols' 333 early_end_msg = 'input data ended while still in quoted string' 334 335 def is_hex(x: int) -> bool: 336 # 48 is ord('0'), 57 is ord('9'), 97 is ord('a'), 102 is ord('f') 337 return 48 <= x <= 57 or 97 <= x <= 102 338 339 def lower(x: int) -> bool: 340 # 65 is ord('A'), 90 is ord('Z') 341 return x + 32 if 65 <= x <= 90 else x 342 343 while True: 344 chunk = r.peek(1) 345 if len(chunk) < 1: 346 raise ValueError(early_end_msg) 347 b = chunk[0] 348 349 if esc: 350 esc = False 351 352 if b == 120: # ord('x') 353 skip_byte(r) 354 chunk = read(r, 2) 355 if len(chunk) != 2: 356 raise ValueError(early_end_msg) 357 a = lower(chunk[0]) 358 b = lower(chunk[1]) 359 w.write(b'\\u00') 360 if not (is_hex(a) and is_hex(b)): 361 raise ValueError(bad_hex_msg) 362 w.write(a) 363 w.write(b) 364 continue 365 366 if b == 117: # ord('u') 367 skip_byte(r) 368 chunk = read(r, 4) 369 if len(chunk) != 4: 370 raise ValueError(early_end_msg) 371 a = lower(chunk[0]) 372 b = lower(chunk[1]) 373 c = lower(chunk[2]) 374 d = lower(chunk[3]) 375 w.write(b'\\u') 376 if not (is_hex(a) and is_hex(b) and is_hex(c) and is_hex(d)): 377 raise ValueError(bad_hex_msg) 378 w.write(chunk) 379 continue 380 381 # numbers for '"', '\\', 'n', 't', 'r', 'b', and 'f' 382 if b in (34, 92, 110, 116, 114, 98, 102): 383 w.write(b'\\') 384 385 w.write(read(r, 1)) 386 continue 387 388 if b == 92: # ord('\\') 389 esc = True 390 skip_byte(r) 391 continue 392 393 if b == quote: 394 skip_byte(r) 395 return 396 397 # emit normal string-byte 398 w.write(read(r, 1)) 399 400 401 def handle_true(w, r) -> None: 402 demand(r, b'true') 403 w.write(b'true') 404 405 406 def handle_True(w, r) -> None: 407 demand(r, b'True') 408 w.write(b'true') 409 410 411 # setup byte-handling lookup tuple 412 bh = [handle_invalid for i in range(256)] 413 bh[ord('0')] = handle_number 414 bh[ord('1')] = handle_number 415 bh[ord('2')] = handle_number 416 bh[ord('3')] = handle_number 417 bh[ord('4')] = handle_number 418 bh[ord('5')] = handle_number 419 bh[ord('6')] = handle_number 420 bh[ord('7')] = handle_number 421 bh[ord('8')] = handle_number 422 bh[ord('9')] = handle_number 423 bh[ord('+')] = handle_positive 424 bh[ord('-')] = handle_negative 425 bh[ord('.')] = handle_dot 426 bh[ord('"')] = handle_double_quoted_string 427 bh[ord('\'')] = handle_single_quoted_string 428 bh[ord('F')] = handle_False 429 bh[ord('N')] = handle_None 430 bh[ord('T')] = handle_True 431 bh[ord('f')] = handle_false 432 bh[ord('n')] = handle_null 433 bh[ord('t')] = handle_true 434 bh[ord('[')] = handle_array 435 bh[ord('(')] = handle_array 436 bh[ord('{')] = handle_object 437 438 # handlers is the immutable byte-driven func-dispatch table 439 handlers = tuple(bh) 440 441 442 def copy_digits(w, r) -> int: 443 'Returns how many digits were copied/handled.' 444 445 copied = 0 446 while True: 447 chunk = r.peek(64) 448 if len(chunk) == 0: 449 return copied 450 451 i = find_digits_end_index(chunk) 452 if i >= 0: 453 w.write(read(r, i)) 454 copied += i 455 return copied 456 else: 457 w.write(chunk) 458 read(r, len(chunk)) 459 copied += len(chunk) 460 461 462 def seek_next_token(r) -> None: 463 'Skip an arbitrarily-long mix of whitespace and comments.' 464 465 while True: 466 chunk = r.peek(1024) 467 if len(chunk) == 0: 468 # input is over, and this func doesn't consider that an error 469 return 470 471 comment = False 472 473 for i, b in enumerate(chunk): 474 # skip space, tab, line-feed, carriage-return, or form-feed 475 if b in (9, 10, 11, 13, 32): 476 continue 477 478 if b == 47 or b == 35: # ord('/'), ord('#') 479 read(r, i) 480 demand_comment(r) 481 comment = True 482 break 483 484 # found start of next token 485 read(r, i) 486 return 487 488 if not comment: 489 read(r, len(chunk)) 490 491 492 def skip_line(r) -> None: 493 while True: 494 chunk = r.peek(1024) 495 if len(chunk) == 0: 496 return 497 498 i = chunk.find(b'\n') 499 if i >= 0: 500 read(r, i + 1) 501 return 502 503 read(r, len(chunk)) 504 505 506 def skip_general_comment(r) -> None: 507 while True: 508 chunk = r.peek(1024) 509 if len(chunk) == 0: 510 raise ValueError(f'input data ended before an expected */') 511 512 i = chunk.find(b'*') 513 if i < 0: 514 # no */ in this chunk, so skip it and try with the next one 515 read(r, len(chunk)) 516 continue 517 518 # skip right past the * just found, then check if a / follows it 519 read(r, i + 1) 520 if peek_byte(r) == 47: # ord('/') 521 # got */, the end of this comment 522 skip_byte(r) 523 return 524 525 526 def find_digits_end_index(chunk: bytes) -> int: 527 i = 0 528 for b in chunk: 529 if 48 <= b <= 57: 530 i += 1 531 else: 532 return i 533 534 # all bytes (if any) were digits, so no end was found 535 return -1 536 537 538 def demand(r, what: bytes) -> None: 539 lead = read(r, len(what)) 540 if not lead.startswith(what): 541 lead = str(lead, encoding='utf-8') 542 what = str(what, encoding='utf-8') 543 raise ValueError(f'expected {what}, but got {lead} instead') 544 545 546 def demand_comment(r) -> None: 547 b = peek_byte(r) 548 if b < 0: 549 raise ValueError('unexpected end of input data') 550 if b == 35: # ord('#') 551 # handle single-line comment 552 skip_line(r) 553 return 554 555 demand(r, b'/') 556 b = peek_byte(r) 557 if b < 0: 558 raise ValueError('unexpected end of input data') 559 560 if b == 47: # ord('/') 561 # handle single-line comment 562 skip_line(r) 563 return 564 565 if b == 42: # ord('*') 566 # handle (potentially) multi-line comment 567 skip_general_comment(r) 568 return 569 570 raise ValueError('expected * or another /, after a /') 571 572 573 def json0(w, src, end) -> None: 574 r = BufferedReader(src) 575 576 # skip leading UTF-8 BOM (byte-order mark) 577 if r.peek(3) == b'\xef\xbb\xbf': 578 read(r, 3) 579 580 # skip leading whitespace/comments 581 seek_next_token(r) 582 583 # emit a single output line, ending with a line-feed 584 b = peek_byte(r) 585 if b >= 0: 586 handlers[b](w, r) 587 else: 588 # w.write(b'null') 589 # treat empty(ish) input as invalid JSON 590 raise ValueError('can\'t turn empty(ish) input into JSON') 591 592 # deliberately run post-processing before checking for trailing-data 593 # errors: for example, if post-proc func emits new line, errors will 594 # show up on their separate line, which is nicer 595 end(w) 596 597 # ignore trailing whitespace/comment bytes, if present 598 seek_next_token(r) 599 600 # ignore trailing semicolon, if present 601 b = peek_byte(r) 602 if b == 59: # ord(';') 603 read(r, 1) 604 # ignore trailing whitespace/comment bytes, if present 605 seek_next_token(r) 606 607 if len(r.peek(1)) > 0: 608 raise ValueError('unexpected trailing bytes in JSON data') 609 610 611 def seems_url(s: str) -> bool: 612 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 613 return any(s.startswith(p) for p in protocols) 614 615 616 def handle_json(w, r) -> None: 617 def end(w) -> None: 618 w.write(b'\n') 619 w.flush() 620 json0(w, r, end) 621 622 623 def handle_json_lines(w, r) -> None: 624 global pos, linenum 625 626 items = 0 627 linenum = 0 628 w.write(b'[') 629 630 while True: 631 line = r.readline().lstrip() 632 if not line: 633 break 634 635 pos = 1 636 linenum += 1 637 638 stripped = line.strip() 639 if not stripped or stripped.startswith(b'//'): 640 continue 641 642 items += 1 643 if items > 1: 644 w.write(b',') 645 646 json0(w, BytesIO(line), lambda w: w.flush()) 647 648 w.write(b']\n') 649 650 651 start_args = 1 652 handle_input = handle_json 653 if len(argv) > 1 and argv[1] in ('-jl', '--jl', '-jsonl', '--jsonl'): 654 start_args = 2 655 handle_input = handle_json_lines 656 657 if len(argv) - 1 > start_args: 658 print(f'multiple inputs not allowed', file=stderr) 659 exit(1) 660 661 w = stdout.buffer 662 name = argv[start_args] if len(argv) > start_args else '-' 663 664 # values keeping track of the input-position, shown in case of errors 665 pos = 1 666 linenum = 1 667 668 try: 669 if name == '-': 670 handle_input(w, stdin.buffer) 671 elif seems_url(name): 672 from urllib.request import urlopen 673 with urlopen(name) as inp: 674 handle_input(w, inp) 675 else: 676 with open(name, mode='rb') as inp: 677 handle_input(w, inp) 678 except BrokenPipeError: 679 # quit quietly, instead of showing a confusing error message 680 stderr.close() 681 exit(0) 682 except KeyboardInterrupt: 683 exit(2) 684 except Exception as e: 685 stdout.write('\n') 686 print(f'line {linenum}, pos {pos} : {e}', file=stderr) 687 exit(1)