File: tlp.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tlp [options...] [python expression] [files/URIs...] 28 29 30 Transform Lines with Python runs a python expression on each line of text 31 input, encoded as UTF-8. Carriage-returns are always ignored in lines, as 32 well as any UTF-8-BOM on the first line of each input. 33 34 The expression can use either `l` or `line` for the current line, and `i` as 35 a 0-based line counter which keeps growing even across input-sources, when 36 given multiple inputs. Also available is `n`, a 1-based line counter which 37 otherwise works the same way. 38 39 Each line is automatically parsed as JSON: when successful, the parsed line 40 is available to the expression as `v`, or `value`. You can check failure to 41 parse JSON by checking if `v` or `value` is of type Skip, since None can be 42 the result of successfully parsing a null JSON value. 43 44 Input-sources can be either files or web-URIs. When not given any explicit 45 named sources, the standard input is used. It's even possible to reuse the 46 standard input using multiple single dashes (-) in the order needed: stdin 47 is only read once in this case, and kept for later reuse. 48 49 When the expression results in None, the current input line is ignored. When 50 the expression results in a boolean, its value determines whether each line 51 is emitted to the standard output, or ignored. 52 53 When the expression emits lists, tuples, or generators, each item is emitted 54 as its own line/result. Since empty containers emit no lines, these are the 55 most general type of results, acting as either filters, or input-amplifiers. 56 57 58 Options 59 60 All options can start with either a single or a double leading dash: 61 62 -h, -help show this help message 63 -m, -mod, -module, -modules import modules named in the next argument, 64 where multiple names are comma-separated 65 -n, -nil, -none, -null don't read any input & run expression once 66 -t, -trace, -traceback turn exceptions into multi-line tracebacks 67 68 An equal-sign `=` (no dashes) argument is also an alias for the `-n` option. 69 70 71 Examples 72 73 # numbers from 0 to 5, each on its own output line; no input is read/used 74 tlp = 'range(6)' 75 76 # all powers up to the 4th, using each input line auto-parsed into a `float` 77 tlp = 'range(1, 6)' | tlp '(v**p for p in range(1, 4+1))' 78 79 # separate input lines with an empty line between each; global var `empty` 80 # can be used to avoid bothering with nested shell-quoting 81 tlp = 'range(6)' | tlp '["", l] if i > 0 else l' 82 83 # ignore errors/exceptions, in favor of the original lines/values 84 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), line)' 85 86 # ignore errors/exceptions, using the current line as a fallback 87 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), line)' 88 89 # ignore errors/exceptions, calling a fallback function with the exception 90 tlp = '("abc", "123")' | tlp 'rescue(lambda: 2 * float(line), str)' 91 92 # transform lines using the format-strings syntax 93 tlp = 'range(50)' | tlp 'f"{line + "**2":>6} = {float(v**2):12,.4f}"' 94 95 # subset/reorder columns from a file with TSV (tab-separated values) lines 96 tlp '"\t".join([tsv[-1], tsv[4], tsv[2]])' data.tsv 97 98 # filtering lines out via None values 99 head -c 1024 /dev/urandom | strings | tlp 'l if len(l) < 20 else None' 100 101 # boolean-valued results are concise ways to filter lines out 102 head -c 1024 /dev/urandom | strings | tlp 'len(l) < 20' 103 104 # function/callable results are automatically called on the current line 105 head -c 1024 /dev/urandom | strings | tlp len 106 107 # emit 10 random integers between 1 and 10 108 tlp -m random = '(random.randint(1, 10) for _ in range(10))' 109 110 # emit standard input lines slowly, delaying output 0.5 seconds each time 111 tlp -m time '(time.sleep(0.5), line)[-1]' 112 113 # timestamp input lines as they become available 114 tlp -m time 'time.strftime("%Y-%m-%d %H:%M:%S\t") + line' 115 116 # show the current date/time 117 tlp -m time = 'time.strftime("%Y-%m-%d %H:%M:%S")' 118 119 # emit documentation for collections.defaultdict from the python stdlib 120 tlp = -m collections 'help(collections.defaultdict)' | cat 121 ''' 122 123 124 from itertools import islice, zip_longest 125 from json import dumps, loads 126 from math import isinf, isnan 127 from re import compile as compile_uncached, IGNORECASE 128 from sys import argv, exit, stderr, stdin, stdout 129 from time import localtime, sleep, strftime 130 from typing import Generator, Iterable 131 132 133 if len(argv) < 2: 134 print(info.strip(), file=stderr) 135 exit(1) 136 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'): 137 print(info.strip()) 138 exit(0) 139 140 141 def handle_no_input(expr): 142 res = eval(expr) 143 144 if isinstance(res, (list, range, set, tuple, Generator)): 145 for e in res: 146 e = adapt_result(e, None) 147 if e is None: 148 continue 149 print(e, flush=flushed) 150 return 151 152 res = adapt_result(res, None) 153 if res is None: 154 return 155 print(res) 156 157 def handle_lines(src, expr): 158 # `comprehension` expressions seem to ignore local variables: even 159 # lambda-based workaround-attempts fail to make needed values like 160 # the current line available to such expressions 161 global i, n, l, line, v, val, value 162 global items, fields, words, tsv 163 164 for l in src: 165 l = l.rstrip('\r\n').rstrip('\n') 166 if i == 0: 167 l = l.lstrip('\xef\xbb\xbf') 168 169 line = l 170 items = fields = words = line.split() 171 tsv = line.split('\t') 172 173 try: 174 v = val = value = loads(l) 175 except BrokenPipeError as e: 176 raise e 177 except Exception as _: 178 v = val = value = Skip() 179 res = eval(expr) 180 181 i += 1 182 n += 1 183 184 if isinstance(res, (list, range, set, tuple, Generator)): 185 n = 0 186 for e in res: 187 e = adapt_result(e, None) 188 if e is None: 189 continue 190 print(e) 191 n += 1 192 if n > 0 and flushed: 193 stdout.flush() 194 n = i + 1 195 continue 196 197 res = adapt_result(res, line) 198 if res is None: 199 continue 200 print(res, flush=flushed) 201 202 def hold_lines(src, lines): 203 for e in src: 204 lines.append(e) 205 yield e 206 207 def adapt_result(res, fallback): 208 if isinstance(res, BaseException): 209 raise res 210 if isinstance(res, Skip) or res is None or res is False: 211 return None 212 if callable(res): 213 return res(fallback) 214 if res is True: 215 return fallback 216 if isinstance(res, dict): 217 return dumps(res, allow_nan=False) 218 return str(res) 219 220 def fail(msg, code = 1): 221 print(str(msg), file=stderr) 222 exit(code) 223 224 def make_open_utf8(open): 225 def open_utf8_readonly(path): 226 return open(path, encoding='utf-8') 227 return open_utf8_readonly 228 229 def seemsurl(path): 230 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 231 return any(path.startswith(p) for p in protocols) 232 233 234 class Skip: 235 def __call__(self, x): 236 return isinstance(x, self.__class__) 237 238 skip = Skip() 239 240 def chunk(items, chunk_size): 241 'Break iterable into chunks, each with up to the item-count given.' 242 243 if isinstance(items, str): 244 n = len(items) 245 while n >= chunk_size: 246 yield items[:chunk_size] 247 items = items[chunk_size:] 248 n -= chunk_size 249 if n > 0: 250 yield items 251 return 252 253 if not isinstance(chunk_size, int): 254 raise Exception('non-integer chunk-size') 255 if chunk_size < 1: 256 raise Exception('non-positive chunk-size') 257 258 it = iter(items) 259 while True: 260 head = tuple(islice(it, chunk_size)) 261 if not head: 262 return 263 yield head 264 265 chunked = chunk 266 267 # re_cache is used by custom function compile to cache previously-compiled 268 # regular-expressions, which makes them quicker to (re)use in formulas 269 re_cache = {} 270 271 def re_compile(expr, flags = 0): 272 'Speed-up using regexes across lines, by avoiding recompilations.' 273 274 if flags in re_cache: 275 cache = re_cache[flags] 276 else: 277 cache = {} 278 re_cache[flags] = cache 279 if expr in cache: 280 return cache[expr] 281 282 pat = compile_uncached(expr, flags) 283 cache[expr] = pat 284 return pat 285 286 def icompile(expr): 287 return re_compile(expr, IGNORECASE) 288 289 def cond(*args): 290 if len(args) == 0: 291 return None 292 293 for i, e in enumerate(args): 294 if i % 2 == 0 and i < len(args) - 1 and e: 295 return args[i + 1] 296 297 return args[-1] if len(args) % 2 == 1 else None 298 299 def dive(into, using): 300 'Depth-first recursive caller for 1-input functions.' 301 302 if callable(into): 303 into, using = using, into 304 305 def rec(v): 306 if isinstance(v, dict): 307 return {k: rec(v) for k, v in v.items()} 308 if isinstance(v, Iterable) and not isinstance(v, str): 309 return [rec(v) for v in v] 310 return using(v) 311 312 return rec(into) 313 314 def divekeys(into, using): 315 'Depth-first recursive caller for 2-input funcs which rename dict keys.' 316 317 if callable(into): 318 into, using = using, into 319 320 def rec(v): 321 if isinstance(v, dict): 322 return {using(k): rec(v) for k, v in v.items()} 323 if isinstance(v, Iterable) and not isinstance(v, str): 324 return [rec(v) for i, v in enumerate(v)] 325 return v 326 327 return rec(None, into) 328 329 def divekv(into, using, using2 = None): 330 'Depth-first recursive caller for 2-input functions.' 331 332 if using2 is None: 333 if callable(into): 334 into, using = using, into 335 else: 336 if not callable(using2): 337 into, using, using2 = using2, into, using 338 339 def rec(k, v): 340 if isinstance(v, dict): 341 return {k: rec(k, v) for k, v in v.items()} 342 if isinstance(v, Iterable) and not isinstance(v, str): 343 return [rec(i, v) for i, v in enumerate(v)] 344 return using(k, v) 345 346 def rec2(k, v): 347 if isinstance(v, dict): 348 return {str(using(k, v)): rec2(k, v) for k, v in v.items()} 349 if isinstance(v, Iterable) and not isinstance(v, str): 350 return [rec2(i, v) for i, v in enumerate(v)] 351 return using2(k, v) 352 353 return rec(None, into) if using2 is None else rec2(None, into) 354 355 kvdive = divekv 356 357 def drop(src, *what): 358 if isinstance(src, str): 359 for s in what: 360 src = src.replace(s, '') 361 return src 362 363 def kdrop(src, what): 364 return {k: v for (k, v) in src.items() if not (k in what)} 365 366 if isinstance(src, dict): 367 return kdrop(src, set(what)) 368 369 if isinstance(src, Iterable): 370 what = set(what) 371 return [kdrop(e, what) for e in src if isinstance(e, dict)] 372 373 return None 374 375 dropped = drop 376 377 def join(x, *y): 378 'Join values into a string, or make a dict from keys and values.' 379 380 if len(y) == 0: 381 return ' '.join(str(v) for v in x) 382 if isinstance(x, str): 383 return x.join(str(v) for v in y) 384 if len(y) == 1 and isinstance(y[0], str): 385 return y[0].join(str(v) for v in x) 386 if len(y) == 1 and isinstance(y[0], (list, range, set, tuple, Generator)): 387 return {k: v for k, v in zip_longest(x, y[0]) if not (k is None)} 388 if isinstance(y, (list, range, set, tuple, Generator)): 389 return {k: v for k, v in zip_longest(x, y) if not (k is None)} 390 return {k: y for k in x} 391 392 def maybe(f, x): 393 try: 394 return f(x) 395 except Exception as _: 396 return x 397 398 def number(x): 399 try: 400 return int(x) 401 except Exception as _: 402 pass 403 try: 404 return float(x) 405 except Exception as _: 406 return x 407 408 def pick(src, *keys): 409 if isinstance(src, dict): 410 return {k: src.get(k, None) for k in keys} 411 return [{k: e.get(k, None) for k in keys} for e in src if isinstance(e, dict)] 412 413 picked = pick 414 415 def plain(s): 416 'Ignore all ANSI-style sequences in a string.' 417 return re_compile('''\x1b\\[([0-9;]+m|[0-9]*[A-HJKST])''').sub('', s) 418 419 def predicate(x): 420 'Helps various higher-order funcs, by standardizing `predicate` values.' 421 if callable(x): 422 return x 423 if isinstance(x, float): 424 if isnan(x): 425 return lambda y: isinstance(y, float) and isnan(y) 426 if isinf(x): 427 return lambda y: isinstance(y, float) and isinf(y) 428 return lambda y: x == y 429 430 def rescue(attempt, fallback = None): 431 try: 432 return attempt() 433 except BrokenPipeError as e: 434 raise e 435 except Exception as e: 436 if callable(fallback): 437 return fallback(e) 438 return fallback 439 440 rescued = rescue 441 442 def retype(x): 443 'Try to narrow the type of the value given.' 444 445 if isinstance(x, float): 446 n = int(x) 447 return n if float(n) == x else x 448 449 if not isinstance(x, str): 450 return x 451 452 try: 453 return loads(x) 454 except Exception: 455 pass 456 457 try: 458 return int(x) 459 except Exception: 460 pass 461 462 try: 463 return float(x) 464 except Exception: 465 pass 466 467 return x 468 469 autocast = autocasted = mold = molded = recast = recasted = remold = retype 470 remolded = retyped = retype 471 472 def json0(x): 473 if isinstance(x, (range, set, Generator)): 474 x = tuple(x) 475 return dumps(x, separators=(',', ':'), allow_nan=False, indent=None) 476 477 j0 = json0 478 479 def jsonl(x): 480 if isinstance(x, Skip): 481 return 482 483 def emit(x): 484 return dumps(x, separators=(', ', ': '), allow_nan=False, indent=None) 485 486 if x is None: 487 yield emit(x) 488 return 489 490 if isinstance(x, (bool, int, float, dict, str)): 491 yield emit(x) 492 return 493 494 if isinstance(x, Iterable): 495 for e in x: 496 if isinstance(e, Skip): 497 continue 498 yield emit(e) 499 return 500 501 yield emit(str(x)) 502 503 jl = jsonlines = ndjson = jsonl 504 505 def typeof(x): 506 return { 507 type(None): 'null', 508 bool: 'boolean', 509 dict: 'object', 510 float: 'number', 511 int: 'number', 512 str: 'string', 513 list: 'array', 514 tuple: 'array', 515 }.get(type(x), 'other') 516 517 jstype = typeof 518 519 def wait(seconds, result): 520 'Wait the given number of seconds, before returning its latter arg.' 521 522 if not isinstance(seconds, (int, float)): 523 if isinstance(result, (int, float)): 524 seconds, result = result, seconds 525 sleep(seconds) 526 return result 527 528 delay = wait 529 530 def after(x, what): 531 i = x.find(what) 532 return '' if i < 0 else x[i+len(what):] 533 534 def afterlast(x, what): 535 i = x.rfind(what) 536 return '' if i < 0 else x[i+len(what):] 537 538 afterfinal = afterlast 539 540 def before(x, what): 541 i = x.find(what) 542 return x if i < 0 else x[:i] 543 544 def beforelast(x, what): 545 i = x.rfind(what) 546 return x if i < 0 else x[:i] 547 548 beforefinal = beforelast 549 550 def since(x, what): 551 i = x.find(what) 552 return '' if i < 0 else x[i:] 553 554 def sincelast(x, what): 555 i = x.rfind(what) 556 return '' if i < 0 else x[i:] 557 558 sincefinal = sincelast 559 560 def until(x, what): 561 i = x.find(what) 562 return x if i < 0 else x[:i+len(what)] 563 564 def untilfinal(x, what): 565 i = x.rfind(what) 566 return x if i < 0 else x[:i+len(what)] 567 568 untillast = untilfinal 569 570 def highlight(s): 571 return f'\x1b[7m{s}\x1b[0m' 572 573 hilite = hilited = highlighted = highlight 574 575 def message(msg, result = None): 576 print(msg, file=stderr) 577 return result 578 579 msg = message 580 581 # seen is used by function `once` to remember previously-given values 582 seen = set() 583 584 def once(x): 585 if x in seen: 586 return None 587 seen.add(x) 588 return x 589 590 dedup = unique = once 591 592 def utf8(x): 593 try: 594 if isinstance(x, str): 595 x = x.encode('utf-8') 596 return str(x, 'utf-8') 597 except Exception: 598 return None 599 600 def ymdhms(when = None): 601 fmt = f'%Y-%m-%d %H:%M:%S' 602 if isinstance(when, (float, int)): 603 return strftime(fmt, localtime(float(when))) 604 if isinstance(when, tuple): 605 return strftime(fmt, when) 606 return strftime(fmt, localtime()) 607 608 609 dquo = dquote = '"' 610 lcurly = '{' 611 rcurly = '}' 612 squo = squote = '\'' 613 614 nil = none = null = None 615 616 617 exec = None 618 open_utf8 = make_open_utf8(open) 619 open = open_utf8 620 621 no_input_opts = ( 622 '=', '-n', '--n', '-nil', '--nil', '-none', '--none', '-null', '--null', 623 ) 624 modules_opts = ( 625 '-m', '--m', '-mod', '--mod', '-module', '--module', 626 '-modules', '--modules', 627 ) 628 trace_opts = ('-t', '--t', '-trace', '--trace', '-traceback', '--traceback') 629 630 args = argv[1:] 631 if any(seemsurl(e) for e in args): 632 from io import TextIOWrapper 633 from urllib.request import urlopen 634 635 no_input = False 636 trace_errors = False 637 638 while len(args) > 0: 639 if args[0] == '--': 640 args = args[1:] 641 break 642 643 if args[0] in no_input_opts: 644 no_input = True 645 args = args[1:] 646 continue 647 648 if args[0] in modules_opts: 649 try: 650 if len(args) < 2: 651 msg = 'a module name or a comma-separated list of modules' 652 raise Exception('expected ' + msg) 653 654 g = globals() 655 from importlib import import_module 656 for e in args[1].split(','): 657 g[e] = import_module(e) 658 659 g = None 660 import_module = None 661 args = args[2:] 662 except Exception as e: 663 fail(e, 1) 664 665 continue 666 667 if args[0] in trace_opts: 668 trace_errors = True 669 args = args[1:] 670 continue 671 672 break 673 674 675 # ensure live-lines output, unless stdout is being saved into a file 676 flushed = stdout.isatty() or not stdout.seekable() 677 678 try: 679 expr = '.' 680 if len(args) > 0: 681 expr = args[0] 682 args = args[1:] 683 684 if expr == '.' and no_input: 685 print(info.strip(), file=stderr) 686 exit(0) 687 688 if expr == '.': 689 expr = 'line' 690 691 expr = compile(expr, expr, mode='eval') 692 compile = None 693 694 if no_input: 695 handle_no_input(expr) 696 exit(0) 697 698 i = 0 699 n = 1 700 v = val = value = Skip() 701 702 items = fields = words = [] 703 tsv = [] 704 705 if len(args) == 0: 706 handle_lines(stdin, expr) 707 exit(0) 708 709 got_stdin = False 710 all_stdin = None 711 dashes = args.count('-') 712 713 for path in args: 714 if path == '-': 715 if dashes > 1: 716 if not got_stdin: 717 all_stdin = [] 718 handle_lines(hold_lines(stdin, all_stdin), expr) 719 got_stdin = True 720 else: 721 handle_lines(all_stdin, expr) 722 else: 723 handle_lines(stdin, expr) 724 continue 725 726 if seemsurl(path): 727 with urlopen(path) as inp: 728 with TextIOWrapper(inp, encoding='utf-8') as txt: 729 handle_lines(txt, expr) 730 continue 731 732 with open_utf8(path) as txt: 733 handle_lines(txt, expr) 734 except BrokenPipeError: 735 # quit quietly, instead of showing a confusing error message 736 stderr.close() 737 exit(0) 738 except KeyboardInterrupt: 739 exit(2) 740 except Exception as e: 741 if trace_errors: 742 raise e 743 else: 744 fail(e, 1)