File: tjp.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tjp [options...] [python expression] [file/URI...] 28 29 30 Transform Json with Python runs a python expression on a single JSON-encoded 31 input. 32 33 The expression can use either `v`, `value`, `d`, or `data` for the decoded 34 input. Invalid-JSON inputs result in an error, with no chance of recovery. 35 36 Input-sources can be either files or web-URIs. When not given a named input, 37 the standard input is used. 38 39 40 Options 41 42 All options can start with either a single or a double leading dash: 43 44 -c, -compact, -j0, -json0 emit result as compact single-line JSON 45 -h, -help show this help message 46 -jl, -jsonl, -ndjson emit JSON Lines when result is an array 47 -m, -mod, -module, -modules import modules named in the next argument, 48 where multiple names are comma-separated 49 -n, -nil, -none, -null don't read any input 50 -t, -trace, -traceback turn exceptions into multi-line tracebacks 51 -z, -zoom zoom into data using rest of the args 52 53 54 Examples 55 56 # numbers from 0 to 5; no input is read/used 57 tjp = 'range(6)' 58 59 # using bases 1 to 5, find all their powers up to the 4th 60 tjp = '((n**p for p in range(1, 4+1)) for n in range(1, 6))' 61 62 # keep only the last 2 items from the input 63 tjp = 'range(1, 6)' | tjp 'data[-2:]' 64 65 # chunk/regroup input items into arrays of up to 3 items each 66 tjp = 'range(1, 8)' | tjp 'chunk(data, 3)' 67 68 # ignore errors/exceptions, in favor of a fallback value 69 tjp = 'rescue(lambda: 2 * float("no way"), "fallback value")' 70 71 # ignore errors/exceptions, calling a fallback func with the exception 72 tjp = 'rescue(lambda: 2 * float("no way"), str)' 73 74 # use dot-syntax on JSON data 75 tjp = '{"abc": {"xyz": 123}}' | tjp -dots 'data.abc.xyz' 76 77 # use dot-syntax on JSON data; keywords as properties are syntax-errors 78 tjp = '{"abc": {"def": 123}}' | tjp -dots 'data.abc["def"]' 79 80 # func results are automatically called on the input 81 tjp = '{"abc": 123, "def": 456}' | tjp len 82 83 # an array of 10 random integers between 1 and 10 84 tjp -m random = '(random.randint(1, 10) for _ in range(10))' 85 86 # try to auto-parse values (esp. numbers) from a table of string values 87 echo '[{"key": "abc", "val": "123"}, {"key": "xyz", "val": "no"}]' | \\ 88 tjp '[{k: rescue(lambda: loads(v), v) for k, v in e.items()} for e in v]' 89 90 # zoom into last array-item of the value of object under `def` key 91 echo '{"abc": 123, "def": [456, 789]}' | tjp -z def -1 92 93 # zoom into last array-item of the last value of object 94 echo '{"abc": 123, "def": [456, 789]}' | tjp -z -1 -1 95 96 # zoom into last value of object 97 echo '{"abc": 123, "def": [456, 789]}' | tjp -z -1 98 99 # zoom into value of object, using key matched case-insensitively 100 echo '{"abc": 123, "def": [456, 789]}' | tjp -z ABC 101 ''' 102 103 104 from itertools import islice, zip_longest 105 from json import dump, load, loads 106 from math import isnan 107 from re import compile as compile_uncached, IGNORECASE 108 from sys import argv, exit, stderr, stdin, stdout 109 from typing import Iterable 110 111 112 if len(argv) < 2: 113 print(info.strip(), file=stderr) 114 exit(1) 115 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'): 116 print(info.strip()) 117 exit(0) 118 119 120 class Skip: 121 def __call__(self, x): 122 return isinstance(x, self.__class__) 123 124 skip = Skip() 125 126 class Dottable: 127 'Enable convenient dot-syntax access to dictionary values.' 128 129 def __getattr__(self, key): 130 return self.__dict__[key] if key in self.__dict__ else None 131 132 def __getitem__(self, key): 133 return self.__dict__[key] if key in self.__dict__ else None 134 135 def __iter__(self): 136 return iter(self.__dict__) 137 138 def dotate(x): 139 'Recursively ensure all dictionaries in a value are dot-accessible.' 140 141 if isinstance(x, dict): 142 d = Dottable() 143 d.__dict__ = {k: dotate(v) for k, v in x.items()} 144 return d 145 if isinstance(x, list): 146 return [dotate(e) for e in x] 147 if isinstance(x, tuple): 148 return tuple(dotate(e) for e in x) 149 return x 150 151 dotated = dote = doted = dotified = dotify = dottified = dottify = dotate 152 153 def chunk(items, chunk_size): 154 'Break iterable into chunks, each with up to the item-count given.' 155 156 if isinstance(items, str): 157 n = len(items) 158 while n >= chunk_size: 159 yield items[:chunk_size] 160 items = items[chunk_size:] 161 n -= chunk_size 162 if n > 0: 163 yield items 164 return 165 166 if not isinstance(chunk_size, int): 167 raise Exception('non-integer chunk-size') 168 if chunk_size < 1: 169 raise Exception('non-positive chunk-size') 170 171 it = iter(items) 172 while True: 173 head = tuple(islice(it, chunk_size)) 174 if not head: 175 return 176 yield head 177 178 chunked = chunk 179 180 # re_cache is used by custom func compile to cache previously-compiled 181 # regular-expressions, which makes them quicker to (re)use in formulas 182 re_cache = {} 183 184 def re_compile(expr, flags = 0): 185 'Speed-up using regexes, by avoiding recompilations.' 186 187 if flags in re_cache: 188 cache = re_cache[flags] 189 else: 190 cache = {} 191 re_cache[flags] = cache 192 if expr in cache: 193 return cache[expr] 194 195 pat = compile_uncached(expr, flags) 196 cache[expr] = pat 197 return pat 198 199 def icompile(expr): 200 return re_compile(expr, IGNORECASE) 201 202 def cond(*args): 203 if len(args) == 0: 204 return None 205 206 for i, e in enumerate(args): 207 if i % 2 == 0 and i < len(args) - 1 and e: 208 return args[i + 1] 209 210 return args[-1] if len(args) % 2 == 1 else None 211 212 def dive(into, using): 213 'Depth-first recursive caller for 1-input functions.' 214 215 if callable(into): 216 into, using = using, into 217 218 def rec(v): 219 if isinstance(v, dict): 220 return {k: rec(v) for k, v in v.items()} 221 if isinstance(v, Iterable) and not isinstance(v, str): 222 return [rec(v) for v in v] 223 return using(v) 224 225 return rec(into) 226 227 def divekeys(into, using): 228 'Depth-first recursive caller for 2-input funcs which rename dict keys.' 229 230 if callable(into): 231 into, using = using, into 232 233 def rec(v): 234 if isinstance(v, dict): 235 return {using(k): rec(v) for k, v in v.items()} 236 if isinstance(v, Iterable) and not isinstance(v, str): 237 return [rec(v) for i, v in enumerate(v)] 238 return v 239 240 return rec(None, into) 241 242 def divekv(into, using, using2 = None): 243 'Depth-first recursive caller for 2-input functions.' 244 245 if using2 is None: 246 if callable(into): 247 into, using = using, into 248 else: 249 if not callable(using2): 250 into, using, using2 = using2, into, using 251 252 def rec(k, v): 253 if isinstance(v, dict): 254 return {k: rec(k, v) for k, v in v.items()} 255 if isinstance(v, Iterable) and not isinstance(v, str): 256 return [rec(i, v) for i, v in enumerate(v)] 257 return using(k, v) 258 259 def rec2(k, v): 260 if isinstance(v, dict): 261 return {str(using(k, v)): rec2(k, v) for k, v in v.items()} 262 if isinstance(v, Iterable) and not isinstance(v, str): 263 # return {str(using(i, v)): rec2(i, v) for i, v in enumerate(v)} 264 return [rec2(i, v) for i, v in enumerate(v)] 265 return using2(k, v) 266 267 return rec(None, into) if using2 is None else rec2(None, into) 268 269 kvdive = divekv 270 271 def drop(src, *what): 272 if isinstance(src, str): 273 for s in what: 274 src = src.replace(s, '') 275 return src 276 277 def kdrop(src, what): 278 return {k: v for (k, v) in src.items() if not (k in what)} 279 280 if isinstance(src, dict): 281 return kdrop(src, set(what)) 282 283 if isinstance(src, Iterable): 284 what = set(what) 285 return [kdrop(e, what) for e in src if isinstance(e, dict)] 286 287 return None 288 289 dropped = drop 290 291 def join(x, *y): 292 'Join values into a string, or make a dict from keys and values.' 293 294 if len(y) == 0: 295 return ' '.join(str(v) for v in x) 296 if isinstance(x, str): 297 return x.join(str(v) for v in y) 298 if len(y) == 1 and isinstance(y[0], str): 299 return y[0].join(str(v) for v in x) 300 if len(y) == 1 and isinstance(y[0], (list, range, set, tuple, Generator)): 301 return {k: v for k, v in zip_longest(x, y[0]) if not (k is None)} 302 if isinstance(y, (list, range, set, tuple, Generator)): 303 return {k: v for k, v in zip_longest(x, y) if not (k is None)} 304 return {k: y for k in x} 305 306 def maybe(f, x): 307 try: 308 return f(x) 309 except Exception as _: 310 return x 311 312 def number(x): 313 try: 314 return int(x) 315 except Exception as _: 316 pass 317 try: 318 return float(x) 319 except Exception as _: 320 return x 321 322 def pick(src, *keys): 323 if isinstance(src, dict): 324 return {k: src.get(k, None) for k in keys} 325 return [{k: e.get(k, None) for k in keys} for e in src if isinstance(e, dict)] 326 327 picked = pick 328 329 def rescue(attempt, fallback = None): 330 try: 331 return attempt() 332 except BrokenPipeError as e: 333 raise e 334 except Exception as e: 335 if callable(fallback): 336 return fallback(e) 337 return fallback 338 339 rescued = rescue 340 341 def retype(x): 342 'Try to narrow the type of the value given.' 343 344 if isinstance(x, float): 345 n = int(x) 346 return n if float(n) == x else x 347 348 if not isinstance(x, str): 349 return x 350 351 try: 352 return loads(x) 353 except Exception: 354 pass 355 356 try: 357 return int(x) 358 except Exception: 359 pass 360 361 try: 362 return float(x) 363 except Exception: 364 pass 365 366 return x 367 368 autocast = autocasted = mold = molded = recast = recasted = remold = retype 369 remolded = retyped = retype 370 371 def typeof(x): 372 return { 373 type(None): 'null', 374 bool: 'boolean', 375 dict: 'object', 376 float: 'number', 377 int: 'number', 378 str: 'string', 379 list: 'array', 380 tuple: 'array', 381 }.get(type(x), 'other') 382 383 jstype = typeof 384 385 386 def result_needs_fixing(x): 387 if isinstance(x, float): 388 return not isnan(x) 389 if x is None or isinstance(x, (bool, int, float, str)): 390 return False 391 rec = result_needs_fixing 392 if isinstance(x, dict): 393 return any(rec(k) or rec(v) for k, v in x.items()) 394 if isinstance(x, (list, tuple)): 395 return any(rec(e) for e in x) 396 return True 397 398 def fix_result(x, default): 399 if x is type: 400 return type(default).__name__ 401 402 if isinstance(x, Skip): 403 return None 404 405 # if expression results in a func, auto-call it with the original data 406 if callable(x): 407 x = x(default) 408 409 if isinstance(x, float) and isnan(x): 410 return None 411 412 if x is None or isinstance(x, (bool, int, float, str)): 413 return x 414 415 rec = fix_result 416 417 if isinstance(x, dict): 418 return { 419 rec(k, default): rec(v, default) for k, v in x.items() if not 420 (isinstance(k, Skip) or isinstance(v, Skip)) 421 } 422 423 if isinstance(x, Iterable): 424 return tuple(rec(e, default) for e in x if not isinstance(e, Skip)) 425 426 if isinstance(x, Dottable): 427 return rec(x.__dict__, default) 428 429 if isinstance(x, Exception): 430 raise x 431 432 return str(x) 433 434 def fail(msg, code = 1): 435 print(str(msg), file=stderr) 436 exit(code) 437 438 def message(msg, result = None): 439 print(msg, file=stderr) 440 return result 441 442 msg = message 443 444 def seemsurl(path): 445 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 446 return any(path.startswith(p) for p in protocols) 447 448 def matchkey(kv, key): 449 if key in kv: 450 return key 451 452 low = key.lower() 453 for k in kv.keys(): 454 if low == k.lower(): 455 return k 456 457 try: 458 i = int(key) 459 l = len(kv) 460 if i < 0: 461 i += l 462 463 if not (-l <= i < l): 464 return key 465 466 for j, k in enumerate(kv.keys()): 467 if i == j: 468 return k 469 except Exception: 470 return key 471 472 return key 473 474 def zoom(data, keys): 475 for i, k in enumerate(keys): 476 if isinstance(data, dict): 477 # m = matchkey(data, k) 478 # if not (m in data): 479 # raise Exception(f'{m}: object doesn\'t have that key') 480 data = data.get(matchkey(data, k), None) 481 continue 482 483 if isinstance(data, (list, tuple)): 484 if k == '+': 485 pick = keys[i + 1:] 486 return [{k: e.get(k, None) for k in pick} 487 for e in data if isinstance(e, dict)] 488 if k == '-': 489 avoid = set(keys[i + 1:]) 490 return [{k: v for (k, v) in e.items() if not (k in avoid)} 491 for e in data if isinstance(e, dict)] 492 if k == '.': 493 rest = keys[i + 1:] 494 return [zoom(e, rest) for e in data] 495 496 try: 497 k = int(k) 498 l = len(data) 499 data = data[k] if -l <= k < l else None 500 except Exception: 501 # raise Exception(f'{k}: arrays don\'t have keys like objects') 502 data = None 503 continue 504 505 # return None 506 # data = None 507 raise Exception(f'{k}: can\'t zoom on value of type {typeof(data)}') 508 509 return data 510 511 def make_eval_once(run): 512 def eval_once(expr): 513 global eval 514 eval = None 515 return run(expr) 516 return eval_once 517 518 519 dquo = dquote = '"' 520 lcurly = '{' 521 rcurly = '}' 522 squo = squote = '\'' 523 524 nil = none = null = None 525 526 527 no_input_opts = ( 528 '=', '-n', '--n', '-nil', '--nil', '-none', '--none', '-null', '--null', 529 ) 530 compact_output_opts = ( 531 '-c', '--c', '-compact', '--compact', '-j0', '--j0', '-json0', '--json0', 532 ) 533 dot_opts = ('--d', '-dot', '--dot', '-dots', '--dots') 534 jsonl_opts = ('-jl', '--jl', '-jsonl', '--jsonl', '-ndjson', '--ndjson') 535 modules_opts = ( 536 '-m', '--m', '-mod', '--mod', '-module', '--module', 537 '-modules', '--modules', 538 ) 539 pipe_opts = ('-p', '--p', '-pipe', '--pipe') 540 trace_opts = ('-t', '--t', '-trace', '--trace', '-traceback', '--traceback') 541 zoom_opts = ('-z', '--z', '-zoom', '--zoom') 542 543 args = argv[1:] 544 no_input = False 545 zoom_stdin = False 546 json_lines = False 547 pipe_mode = False 548 trace_errors = False 549 dottable_input = False 550 compact_output = False 551 552 while len(args) > 0: 553 if args[0] == '--': 554 args = args[1:] 555 break 556 557 if args[0] in no_input_opts: 558 no_input = True 559 args = args[1:] 560 continue 561 562 if args[0] in compact_output_opts: 563 compact_output = True 564 args = args[1:] 565 continue 566 567 if args[0] in dot_opts: 568 dottable_input = True 569 args = args[1:] 570 continue 571 572 if args[0] in jsonl_opts: 573 json_lines = True 574 args = args[1:] 575 continue 576 577 if args[0] in pipe_opts: 578 pipe_mode = True 579 args = args[1:] 580 break 581 582 if args[0] in modules_opts: 583 try: 584 if len(args) < 2: 585 msg = 'a module name or a comma-separated list of modules' 586 raise Exception('expected ' + msg) 587 588 g = globals() 589 from importlib import import_module 590 for e in args[1].split(','): 591 g[e] = import_module(e) 592 593 g = None 594 import_module = None 595 args = args[2:] 596 except Exception as e: 597 fail(e, 1) 598 599 continue 600 601 if args[0] in trace_opts: 602 trace_errors = True 603 args = args[1:] 604 continue 605 606 if args[0] in zoom_opts: 607 zoom_stdin = True 608 args = args[1:] 609 break 610 611 break 612 613 614 try: 615 if zoom_stdin: 616 data = load(stdin) 617 data = zoom(data, args) 618 v = data 619 else: 620 expr = 'data' 621 if len(args) > 0 and (not pipe_mode): 622 expr = args[0] 623 args = args[1:] 624 625 if expr == '.': 626 expr = 'data' 627 if not pipe_mode: 628 expr = compile(expr, expr, mode='eval') 629 630 if (not pipe_mode) and len(args) > 1: 631 raise Exception('can\'t use more than 1 input') 632 path = '-' if len(args) == 0 or pipe_mode else args[0] 633 634 if no_input: 635 data = None 636 elif path == '-': 637 data = load(stdin) 638 elif seemsurl(path): 639 from io import TextIOWrapper 640 from urllib.request import urlopen 641 with urlopen(path) as inp: 642 with TextIOWrapper(inp, encoding='utf-8') as txt: 643 data = load(txt) 644 else: 645 with open(path, encoding='utf-8') as inp: 646 data = load(inp) 647 648 if dottable_input: 649 data = dotate(data) 650 651 v = val = value = d = dat = data 652 exec = None 653 open = None 654 compile = None 655 656 if pipe_mode: 657 funcs = [eval(s) for s in args] 658 eval = None 659 660 # variable names `o` and `p` work like in the `pyp` tool, except 661 # the pipeline steps were given as separate cmd-line arguments 662 global o, p 663 664 o = p = prev = v 665 for f in funcs: 666 p = f(p) 667 if callable(p): 668 p = p(prev) 669 prev = p 670 v = p 671 else: 672 eval = make_eval_once(eval) 673 v = eval(expr) 674 675 if result_needs_fixing(v): 676 v = fix_result(v, value) 677 678 seps = (',', ':') if compact_output else (',', ': ') 679 680 if json_lines and isinstance(v, (list, tuple)): 681 for e in v: 682 dump(e, stdout, indent=None, separators=seps, allow_nan=False) 683 stdout.write('\n') 684 else: 685 indent = None if compact_output or json_lines else 2 686 dump(v, stdout, indent=indent, separators=seps, allow_nan=False) 687 stdout.write('\n') 688 except BrokenPipeError: 689 # quit quietly, instead of showing a confusing error message 690 stderr.close() 691 exit(0) 692 except KeyboardInterrupt: 693 exit(2) 694 except Exception as e: 695 if trace_errors: 696 raise e 697 else: 698 fail(e, 1)