File: tlp.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tlp [options...] [python expression] [files/URIs...] 28 29 30 Transform Lines with Python runs a python expression on each line of text 31 input, encoded as UTF-8. Carriage-returns are always ignored in lines, as 32 well as any UTF-8-BOM on the first line of each input. 33 34 The expression can use either `l` or `line` for the current line, and `i` as 35 a 0-based line counter which keeps growing even across input-sources, when 36 given multiple inputs. Also available is `n`, a 1-based line counter which 37 otherwise works the same way. 38 39 Each line is automatically parsed as JSON: when successful, the parsed line 40 is available to the expression as `v`, or `value`. You can check failure to 41 parse JSON by checking if `v` or `value` is of type Skip, since None can be 42 the result of successfully parsing a null JSON value. 43 44 Input-sources can be either files or web-URIs. When not given any explicit 45 named sources, the standard input is used. It's even possible to reuse the 46 standard input using multiple single dashes (-) in the order needed: stdin 47 is only read once in this case, and kept for later reuse. 48 49 When the expression results in None, the current input line is ignored. When 50 the expression results in a boolean, its value determines whether each line 51 is emitted to the standard output, or ignored. 52 53 When the expression emits lists, tuples, or generators, each item is emitted 54 as its own line/result. Since empty containers emit no lines, these are the 55 most general type of results, acting as either filters, or input-amplifiers. 56 57 58 Options 59 60 All options can start with either a single or a double leading dash: 61 62 -add, -sum, -total sum numeric results, show at the end 63 -c, -count count True-like results, show at the end 64 -h, -help show this help message 65 -m, -mod, -module, -modules import modules named in the next argument, 66 where multiple names are comma-separated 67 -n, -nil, -none, -null don't read any input & run expression once 68 -t, -trace, -traceback turn exceptions into multi-line tracebacks 69 70 An equal-sign `=` (no dashes) argument is also an alias for the `-n` option. 71 72 73 Examples 74 75 # numbers from 0 to 5, each on its own output line; no input is read/used 76 tlp = 'range(6)' 77 78 # all powers up to the 4th, using each input line auto-parsed into a `float` 79 tlp = 'range(1, 6)' | tlp '(v**p for p in range(1, 4 + 1))' 80 81 # separate input lines with an empty line between each; global var `empty` 82 # can be used to avoid bothering with nested shell-quoting 83 tlp = 'range(6)' | tlp '["", l] if i > 0 else l' 84 85 # ignore errors/exceptions, in favor of the original lines/values 86 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), line)' 87 88 # ignore errors/exceptions, using the current line as a fallback 89 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), line)' 90 91 # ignore errors/exceptions, calling a fallback function with the exception 92 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), str)' 93 94 # transform lines using the format-strings syntax 95 tlp = 'range(50)' | tlp 'f"{line + "**2":>6} = {float(v**2):12,.4f}"' 96 97 # subset/reorder columns from a file with TSV (tab-separated values) lines 98 tlp '"\t".join([tsv[-1], tsv[4], tsv[2]])' data.tsv 99 100 # filtering lines out via None values 101 head -c 1024 /dev/urandom | strings | tlp 'l if len(l) < 20 else None' 102 103 # boolean-valued results are concise ways to filter lines out 104 head -c 1024 /dev/urandom | strings | tlp 'len(l) < 20' 105 106 # function/callable results are automatically called on the current line 107 head -c 1024 /dev/urandom | strings | tlp len 108 109 # emit 10 random integers between 1 and 10 110 tlp -m random = '(random.randint(1, 10) for _ in range(10))' 111 112 # emit standard input lines slowly, delaying output 0.5 seconds each time 113 tlp -m time '(time.sleep(0.5), line)[-1]' 114 115 # timestamp input lines as they become available 116 tlp -m time 'time.strftime("%Y-%m-%d %H:%M:%S\t") + line' 117 118 # show the current date/time 119 tlp -m time = 'time.strftime("%Y-%m-%d %H:%M:%S")' 120 121 # emit documentation for collections.defaultdict from the python stdlib 122 tlp = -m collections 'help(collections.defaultdict)' | cat 123 ''' 124 125 126 from itertools import islice, zip_longest 127 from json import dumps, loads 128 from math import isinf, isnan 129 from re import compile as compile_uncached, IGNORECASE 130 from sys import argv, exit, stderr, stdin, stdout 131 from time import localtime, sleep, strftime 132 from typing import Generator, Iterable 133 134 135 if len(argv) < 2: 136 print(info.strip(), file=stderr) 137 exit(1) 138 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'): 139 print(info.strip()) 140 exit(0) 141 142 143 def handle_no_input(expr): 144 res = eval(expr) 145 146 if isinstance(res, (list, range, set, tuple, Generator)): 147 for e in res: 148 e = adapt_result(e, None) 149 if e is None: 150 continue 151 print(e, flush=flushed) 152 return 153 154 res = adapt_result(res, None) 155 if res is None: 156 return 157 print(res) 158 159 def handle_lines(src, expr): 160 global adding, counting, ok, total 161 162 # `comprehension` expressions seem to ignore local variables: even 163 # lambda-based workaround-attempts fail to make needed values like 164 # the current line available to such expressions 165 global i, n, fn, l, line, v, val, value 166 global items, fields, words, tsv 167 168 fn = 1 169 170 for l in src: 171 l = l.rstrip('\r\n').rstrip('\n') 172 if fn == 1: 173 l = l.lstrip('\xef\xbb\xbf') 174 175 line = l 176 items = fields = words = line.split() 177 tsv = line.split('\t') 178 179 try: 180 v = val = value = loads(l) 181 except BrokenPipeError as e: 182 raise e 183 except Exception as _: 184 v = val = value = Skip() 185 res = eval(expr) 186 187 i += 1 188 n += 1 189 fn += 1 190 191 try: 192 ok += bool(res) if counting else 1 193 total += res 194 except Exception as _: 195 pass 196 197 if adding or counting: 198 continue 199 200 if isinstance(res, (list, range, set, tuple, Generator)): 201 n = 0 202 for e in res: 203 e = adapt_result(e, None) 204 if e is None: 205 continue 206 print(e) 207 n += 1 208 if n > 0 and flushed: 209 stdout.flush() 210 n = i + 1 211 continue 212 213 res = adapt_result(res, line) 214 if res is None: 215 continue 216 print(res, flush=flushed) 217 218 def hold_lines(src, lines): 219 for e in src: 220 lines.append(e) 221 yield e 222 223 def adapt_result(res, fallback): 224 if isinstance(res, BaseException): 225 raise res 226 if isinstance(res, Skip) or res is None or res is False: 227 return None 228 if callable(res): 229 return res(fallback) 230 if res is True: 231 return fallback 232 if isinstance(res, dict): 233 return dumps(res, allow_nan=False) 234 return str(res) 235 236 def fail(msg, code = 1): 237 print(str(msg), file=stderr) 238 exit(code) 239 240 def make_open_utf8(open): 241 def open_utf8_readonly(path): 242 return open(path, encoding='utf-8') 243 return open_utf8_readonly 244 245 def seemsurl(path): 246 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 247 return any(path.startswith(p) for p in protocols) 248 249 250 class Skip: 251 def __call__(self, x): 252 return isinstance(x, self.__class__) 253 254 skip = Skip() 255 256 def cases(x, *args): 257 ''' 258 Simulate a switch statement on a value, using matches/result pairs from 259 the arguments given; when given an even number of extra args, None is 260 used as a final fallback result; when given an odd number of extra args, 261 the last argument is used as a final `default` value, if needed. 262 ''' 263 264 for i in range(0, len(args) - len(args) % 2, 2): 265 test, res = args[i], args[i+1] 266 if isinstance(test, (list, tuple)) and x in test: 267 return res 268 if isinstance(test, float) and isnan(test) and isnan(x): 269 return res 270 if x == test: 271 return res 272 return None if len(args) % 2 == 0 else args[-1] 273 274 switch = cases 275 276 def chunk(items, chunk_size): 277 'Break iterable into chunks, each with up to the item-count given.' 278 279 if isinstance(items, str): 280 n = len(items) 281 while n >= chunk_size: 282 yield items[:chunk_size] 283 items = items[chunk_size:] 284 n -= chunk_size 285 if n > 0: 286 yield items 287 return 288 289 if not isinstance(chunk_size, int): 290 raise Exception('non-integer chunk-size') 291 if chunk_size < 1: 292 raise Exception('non-positive chunk-size') 293 294 it = iter(items) 295 while True: 296 head = tuple(islice(it, chunk_size)) 297 if not head: 298 return 299 yield head 300 301 chunked = chunk 302 303 # re_cache is used by custom function compile to cache previously-compiled 304 # regular-expressions, which makes them quicker to (re)use in formulas 305 re_cache = {} 306 307 def re_compile(expr, flags = 0): 308 'Speed-up using regexes across lines, by avoiding recompilations.' 309 310 if flags in re_cache: 311 cache = re_cache[flags] 312 else: 313 cache = {} 314 re_cache[flags] = cache 315 if expr in cache: 316 return cache[expr] 317 318 pat = compile_uncached(expr, flags) 319 cache[expr] = pat 320 return pat 321 322 def icompile(expr): 323 return re_compile(expr, IGNORECASE) 324 325 def compose(*what): 326 def composite(x): 327 for f in what: 328 x = f(x) 329 return x 330 return composite 331 332 def cond(*args): 333 for i, e in enumerate(args): 334 if i % 2 == 0 and i < len(args) - 1 and e: 335 return args[i + 1] 336 337 return args[-1] if len(args) % 2 == 1 else None 338 339 def denan(x, fallback = None): 340 'Replace floating-point NaN with the alternative value given.' 341 return x if not (isinstance(x, float) and isnan(x)) else fallback 342 343 def denil(*args): 344 'Avoid None values, if possible: first value which isn\'t None wins.' 345 for e in args: 346 if e != None: 347 return e 348 return None 349 350 denone = denull = denil 351 352 def dive(into, using): 353 'Depth-first recursive caller for 1-input functions.' 354 355 if callable(into): 356 into, using = using, into 357 358 def rec(v): 359 if isinstance(v, dict): 360 return {k: rec(v) for k, v in v.items()} 361 if isinstance(v, Iterable) and not isinstance(v, str): 362 return [rec(v) for v in v] 363 return using(v) 364 365 return rec(into) 366 367 def divekeys(into, using): 368 'Depth-first recursive caller for 2-input funcs which rename dict keys.' 369 370 if callable(into): 371 into, using = using, into 372 373 def rec(v): 374 if isinstance(v, dict): 375 return {using(k): rec(v) for k, v in v.items()} 376 if isinstance(v, Iterable) and not isinstance(v, str): 377 return [rec(v) for i, v in enumerate(v)] 378 return v 379 380 return rec(None, into) 381 382 def divekv(into, using, using2 = None): 383 'Depth-first recursive caller for 2-input functions.' 384 385 if using2 is None: 386 if callable(into): 387 into, using = using, into 388 else: 389 if not callable(using2): 390 into, using, using2 = using2, into, using 391 392 def rec(k, v): 393 if isinstance(v, dict): 394 return {k: rec(k, v) for k, v in v.items()} 395 if isinstance(v, Iterable) and not isinstance(v, str): 396 return [rec(i, v) for i, v in enumerate(v)] 397 return using(k, v) 398 399 def rec2(k, v): 400 if isinstance(v, dict): 401 return {str(using(k, v)): rec2(k, v) for k, v in v.items()} 402 if isinstance(v, Iterable) and not isinstance(v, str): 403 return [rec2(i, v) for i, v in enumerate(v)] 404 return using2(k, v) 405 406 return rec(None, into) if using2 is None else rec2(None, into) 407 408 kvdive = divekv 409 410 def drop(src, *what): 411 if isinstance(src, str): 412 for s in what: 413 src = src.replace(s, '') 414 return src 415 416 def kdrop(src, what): 417 return {k: v for (k, v) in src.items() if not (k in what)} 418 419 if isinstance(src, dict): 420 return kdrop(src, set(what)) 421 422 if isinstance(src, Iterable): 423 what = set(what) 424 return [kdrop(e, what) for e in src if isinstance(e, dict)] 425 426 return None 427 428 dropped = drop 429 430 def flat(*args): 431 'Turn arbitrarily-nested values/sequences into a single flat sequence.' 432 433 def rec(x): 434 if x is None: 435 return 436 437 if isinstance(x, dict): 438 yield from rec(x.values()) 439 440 if isinstance(x, str): 441 yield x 442 return 443 444 if isinstance(x, Iterable): 445 for e in x: 446 yield from rec(e) 447 return 448 449 yield x 450 451 for x in args: 452 yield from rec(x) 453 454 flatten = flattened = flat 455 456 def identity(x): 457 return x 458 459 def iota(n): 460 return range(1, n + 1) 461 462 def join(x, *y): 463 'Join values into a string, or make a dict from keys and values.' 464 465 if len(y) == 0: 466 return ' '.join(str(v) for v in x) 467 if isinstance(x, str): 468 return x.join(str(v) for v in y) 469 if len(y) == 1 and isinstance(y[0], str): 470 return y[0].join(str(v) for v in x) 471 if len(y) == 1 and isinstance(y[0], (list, range, set, tuple, Generator)): 472 return {k: v for k, v in zip_longest(x, y[0]) if not (k is None)} 473 if isinstance(y, (list, range, set, tuple, Generator)): 474 return {k: v for k, v in zip_longest(x, y) if not (k is None)} 475 return {k: y for k in x} 476 477 joined = join 478 479 def maybe(f, x = None): 480 try: 481 return f(x) 482 except Exception as _: 483 return x 484 485 def number(x): 486 try: 487 return int(x) 488 except Exception as _: 489 pass 490 try: 491 return float(x) 492 except Exception as _: 493 return x 494 495 def pick(src, *keys): 496 if isinstance(src, dict): 497 return {k: src.get(k, None) for k in keys} 498 return [{k: e.get(k, None) for k in keys} for e in src if isinstance(e, dict)] 499 500 picked = pick 501 502 def plain(s): 503 'Ignore all ANSI-style sequences in a string.' 504 return re_compile('''\x1b\\[([0-9;]+m|[0-9]*[A-HJKST])''').sub('', s) 505 506 def predicate(x): 507 'Helps various higher-order funcs, by standardizing `predicate` values.' 508 if callable(x): 509 return x 510 if not isinstance(x, float): 511 return lambda y: x == y 512 if isnan(x): 513 return lambda y: isinstance(y, float) and isnan(y) 514 if isinf(x): 515 return lambda y: isinstance(y, float) and isinf(y) 516 517 def rescue(attempt, fallback = None): 518 try: 519 return attempt() 520 except BrokenPipeError as e: 521 raise e 522 except Exception as e: 523 if callable(fallback): 524 return fallback(e) 525 return fallback 526 527 rescued = rescue 528 529 def retype(x): 530 'Try to narrow the type of the value given.' 531 532 if isinstance(x, float): 533 n = int(x) 534 return n if float(n) == x else x 535 536 if not isinstance(x, str): 537 return x 538 539 try: 540 return loads(x) 541 except Exception as _: 542 pass 543 544 try: 545 return int(x) 546 except Exception as _: 547 pass 548 549 try: 550 return float(x) 551 except Exception as _: 552 pass 553 554 return x 555 556 autocast = autocasted = mold = molded = recast = recasted = remold = retype 557 remolded = retyped = retype 558 559 def json0(x): 560 if isinstance(x, (range, set, Generator)): 561 x = tuple(x) 562 return dumps(x, separators=(',', ':'), allow_nan=False, indent=None) 563 564 j0 = json0 565 566 def jsonl(x): 567 if isinstance(x, Skip): 568 return 569 570 def emit(x): 571 return dumps(x, separators=(', ', ': '), allow_nan=False, indent=None) 572 573 if x is None: 574 yield emit(x) 575 return 576 577 if isinstance(x, (bool, int, float, dict, str)): 578 yield emit(x) 579 return 580 581 if isinstance(x, Iterable): 582 for e in x: 583 if not isinstance(e, Skip): 584 yield emit(e) 585 return 586 587 yield emit(str(x)) 588 589 jl = jsonlines = ndjson = jsonl 590 591 def typeof(x): 592 return { 593 type(None): 'null', 594 bool: 'boolean', 595 dict: 'object', 596 float: 'number', int: 'number', 597 str: 'string', 598 list: 'array', tuple: 'array', 599 }.get(type(x), 'other') 600 601 jstype = typeof 602 603 def wait(seconds, result): 604 'Wait the given number of seconds, before returning its latter arg.' 605 606 if not isinstance(seconds, (int, float)): 607 if isinstance(result, (int, float)): 608 seconds, result = result, seconds 609 sleep(seconds) 610 return result 611 612 delay = wait 613 614 def after(x, what): 615 i = x.find(what) 616 return '' if i < 0 else x[i+len(what):] 617 618 def afterlast(x, what): 619 i = x.rfind(what) 620 return '' if i < 0 else x[i+len(what):] 621 622 afterfinal = afterlast 623 624 def before(x, what): 625 i = x.find(what) 626 return x if i < 0 else x[:i] 627 628 def beforelast(x, what): 629 i = x.rfind(what) 630 return x if i < 0 else x[:i] 631 632 beforefinal = beforelast 633 634 def since(x, what): 635 i = x.find(what) 636 return '' if i < 0 else x[i:] 637 638 def sincelast(x, what): 639 i = x.rfind(what) 640 return '' if i < 0 else x[i:] 641 642 sincefinal = sincelast 643 644 def until(x, what): 645 i = x.find(what) 646 return x if i < 0 else x[:i+len(what)] 647 648 def untilfinal(x, what): 649 i = x.rfind(what) 650 return x if i < 0 else x[:i+len(what)] 651 652 untillast = untilfinal 653 654 def highlight(s): 655 return f'\x1b[7m{s}\x1b[0m' 656 657 hilite = hilited = highlighted = highlight 658 659 def message(msg, result = None): 660 print(msg, file=stderr) 661 return result 662 663 msg = message 664 665 # seen is used by function `once` to remember previously-given values 666 seen = set() 667 668 def once(x): 669 if x in seen: 670 return None 671 seen.add(x) 672 return x 673 674 dedup = unique = once 675 676 def underline(s): 677 return f'\x1b[4m{s}\x1b[0m' 678 679 underlined = underline 680 681 def utf8(x): 682 try: 683 if isinstance(x, str): 684 x = x.encode('utf-8') 685 return str(x, 'utf-8') 686 except Exception as _: 687 return None 688 689 def ymdhms(when = None): 690 fmt = f'%Y-%m-%d %H:%M:%S' 691 if isinstance(when, (float, int)): 692 return strftime(fmt, localtime(float(when))) 693 if isinstance(when, tuple): 694 return strftime(fmt, when) 695 return strftime(fmt, localtime()) 696 697 698 amp = '&' 699 apos = '\'' 700 lcur = lcurly = '{' 701 quot = '"' 702 rcur = rcurly = '}' 703 704 true = True 705 false = False 706 nil = none = null = None 707 708 709 exec = None 710 open_utf8 = make_open_utf8(open) 711 open = open_utf8 712 713 adding_opts = ('-add', '--add', '-sum', '--sum') 714 counting_opts = ('-c', '--c', '-count', '--count') 715 no_input_opts = ( 716 '=', '-n', '--n', '-nil', '--nil', '-none', '--none', '-null', '--null', 717 ) 718 modules_opts = ( 719 '-m', '--m', '-mod', '--mod', '-module', '--module', 720 '-modules', '--modules', 721 ) 722 tracing_opts = ('-t', '--t', '-trace', '--trace', '-traceback', '--traceback') 723 724 args = argv[1:] 725 if any(seemsurl(e) for e in args): 726 from io import TextIOWrapper 727 from urllib.request import urlopen 728 729 no_input = False 730 adding = False 731 counting = False 732 tracing = False 733 734 while len(args) > 0: 735 if args[0] == '--': 736 args = args[1:] 737 break 738 739 if args[0] in adding_opts: 740 adding = True 741 args = args[1:] 742 continue 743 744 if args[0] in counting_opts: 745 counting = True 746 args = args[1:] 747 continue 748 749 if args[0] in no_input_opts: 750 no_input = True 751 args = args[1:] 752 continue 753 754 if args[0] in modules_opts: 755 try: 756 if len(args) < 2: 757 msg = 'a module name or a comma-separated list of modules' 758 raise Exception('expected ' + msg) 759 760 from importlib import import_module 761 for e in args[1].split(','): 762 globals()[e] = import_module(e) 763 764 import_module = None 765 args = args[2:] 766 except Exception as e: 767 fail(e, 1) 768 769 continue 770 771 if args[0] in tracing_opts: 772 tracing = True 773 args = args[1:] 774 continue 775 776 if args[0] != '-' and args[0].startswith('-'): 777 fail(f'tlp: unsupported option {args[0]}', 1) 778 779 break 780 781 782 # ensure live-lines output, unless stdout is being saved into a file 783 flushed = stdout.isatty() or not stdout.seekable() 784 785 expr = '.' 786 if len(args) > 0: 787 expr = args[0] 788 args = args[1:] 789 790 if expr == '.' and no_input: 791 print(info.strip(), file=stderr) 792 exit(1) 793 794 if expr == '.': 795 expr = 'line' 796 797 try: 798 expr = compile(expr, expr, mode='eval') 799 compile = None 800 801 if no_input: 802 handle_no_input(expr) 803 exit(0) 804 805 i = 0 806 n = 1 807 v = val = value = Skip() 808 809 items = fields = words = [] 810 tsv = [] 811 812 ok = 0 813 total = 0 814 got_stdin = False 815 stdin_lines = None 816 hold_stdin = args.count('-') > 1 817 818 for path in args: 819 filename = pathname = path 820 821 if path == '-': 822 if not hold_stdin: 823 handle_lines(stdin, expr) 824 continue 825 if got_stdin: 826 handle_lines(stdin_lines, expr) 827 continue 828 stdin_lines = [] 829 handle_lines(hold_lines(stdin, stdin_lines), expr) 830 got_stdin = True 831 continue 832 833 if seemsurl(path): 834 with urlopen(path) as inp: 835 with TextIOWrapper(inp, encoding='utf-8') as txt: 836 handle_lines(txt, expr) 837 continue 838 839 with open_utf8(path) as txt: 840 handle_lines(txt, expr) 841 842 if len(args) == 0: 843 handle_lines(stdin, expr) 844 845 if adding and counting: 846 print('count\t' + str(ok)) 847 print('total\t' + str(total)) 848 elif adding: 849 print(total) 850 elif counting: 851 print(ok) 852 except BrokenPipeError: 853 # quit quietly, instead of showing a confusing error message 854 stderr.close() 855 exit(0) 856 except KeyboardInterrupt: 857 exit(2) 858 except Exception as e: 859 if tracing: 860 raise e 861 fail(e, 1)