File: groupy.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 groupy [options...] [python expression] [files/URIs...] 28 29 30 GROUp with PYthon groups/ties lines together by running the Python expression 31 given on each line, and using its common results. Inputs are encoded as UTF-8. 32 Carriage-returns are always ignored in lines, as well as any UTF-8-BOM on the 33 first line of each input. 34 35 The expression can use either `l` or `line` for the current line, and `i` as 36 a 0-based line counter which keeps growing even across input-sources, when 37 given multiple inputs. Also available is `n`, a 1-based line counter which 38 otherwise works the same way. 39 40 Each line is automatically parsed as JSON: when successful, the parsed line 41 is available to the expression as `v`, or `value`. You can check failure to 42 parse JSON by checking if `v` or `value` is of type Skip, since None can be 43 the result of successfully parsing a null JSON value. 44 45 Input-sources can be either files or web-URIs. When not given any explicit 46 named sources, the standard input is used. A single dash also means the 47 standard input. 48 49 When the expression results in None, the current input line is ignored. 50 51 52 Options 53 54 All options can start with either a single or a double leading dash: 55 56 -c, -compact, -j0, -json0 emit a single-line of compact JSON instead 57 -h, -help show this help message 58 -j, -json emit multi-line JSON, instead of lines 59 -m, -mod, -module, -modules import modules named in the next argument, 60 where multiple names are comma-separated 61 -t, -trace, -traceback turn exceptions into multi-line tracebacks 62 63 64 Examples 65 66 # read from a file, and only emit lines exceeding 80 items 67 groupy '1 if len(line) > 80 else None' data.txt 68 69 # group files in current folder (roughly) by common auto-detected file-type 70 file * | groupy '(lambda e: e[1] + e[2])(" ".join(l.split(":")[1:]).split())' 71 72 # group files in current folder (roughly) by common auto-detected file-type 73 file * | groupy 'fields[1] if len(fields) < 3 else fields[1] + fields[2]' 74 75 # group files by auto-detected file-type, emitting JSON output instead 76 file * | groupy -j 'items[1] if len(items) < 3 else f"{items[1]} {items[2]}"' 77 78 # group numbers by common remainder when divided by 5 79 seq 50 | groupy 'v % 5' 80 81 # transform lines in order with a format-string, without grouping anything 82 seq 50 | groupy 'print(f"{line + "**2":>6} = {float(v**2):12,.4f}")' 83 84 # group numbers by common remainder when divided by 5, ignoring numbers > 30 85 seq 50 | groupy 'None if v > 30 else v % 5' 86 87 # alternate lines into 3 rotating groups, limiting each group-size to 5 88 seq 500 | groupy 'n % 3 if len(groups.get(n % 3, [])) < 5 else None' 89 ''' 90 91 92 from itertools import islice, zip_longest 93 from json import dump, dumps, loads 94 from math import isinf, isnan 95 from re import compile as compile_uncached, IGNORECASE 96 from sys import argv, exit, stderr, stdin, stdout 97 from time import localtime, strftime 98 from typing import Iterable 99 100 101 if len(argv) < 2: 102 print(info.strip(), file=stderr) 103 exit(1) 104 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'): 105 print(info.strip()) 106 exit(0) 107 108 109 def group_lines(src, expr): 110 global first_key 111 112 # `comprehension` expressions seem to ignore local variables: even 113 # lambda-based workaround-attempts fail to make needed values like 114 # the current line available to such expressions 115 global i, n, l, line, v, val, value 116 global groups, items, fields, words, tsv 117 118 for l in src: 119 l = l.rstrip('\r\n').rstrip('\n') 120 if i == 0: 121 l = l.lstrip('\xef\xbb\xbf') 122 123 line = l 124 items = fields = words = line.split() 125 tsv = line.split('\t') 126 127 try: 128 v = val = value = loads(l) 129 except Exception as _: 130 v = val = value = Skip() 131 132 k = eval(expr) 133 if callable(k): 134 k = k(line) 135 if isinstance(k, BaseException): 136 raise k 137 138 i += 1 139 n += 1 140 141 if k is None or isinstance(k, Skip): 142 continue 143 if first_key is None: 144 first_key = k 145 146 if k in groups: 147 groups[k].append(line) 148 else: 149 groups[k] = [line] 150 151 # emit lines for the first group right away, since its items are 152 # available as soon as they come from the input 153 if k == first_key: 154 print(line, flush=flushed) 155 156 def fail(msg, code = 1): 157 print(str(msg), file=stderr) 158 exit(code) 159 160 def make_open_utf8(open): 161 def open_utf8_readonly(path): 162 return open(path, encoding='utf-8') 163 return open_utf8_readonly 164 165 def seemsurl(path): 166 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 167 return any(path.startswith(p) for p in protocols) 168 169 class Skip: 170 def __call__(self, x): 171 return isinstance(x, self.__class__) 172 173 skip = Skip() 174 175 def chunk(items, chunk_size): 176 'Break iterable into chunks, each with up to the item-count given.' 177 178 if isinstance(items, str): 179 n = len(items) 180 while n >= chunk_size: 181 yield items[:chunk_size] 182 items = items[chunk_size:] 183 n -= chunk_size 184 if n > 0: 185 yield items 186 return 187 188 if not isinstance(chunk_size, int): 189 raise Exception('non-integer chunk-size') 190 if chunk_size < 1: 191 raise Exception('non-positive chunk-size') 192 193 it = iter(items) 194 while True: 195 head = tuple(islice(it, chunk_size)) 196 if not head: 197 return 198 yield head 199 200 chunked = chunk 201 202 # re_cache is used by custom func compile to cache previously-compiled 203 # regular-expressions, which makes them quicker to (re)use in formulas 204 re_cache = {} 205 206 def re_compile(expr, flags = 0): 207 'Speed-up using regexes across lines, by avoiding recompilations.' 208 209 if flags in re_cache: 210 cache = re_cache[flags] 211 else: 212 cache = {} 213 re_cache[flags] = cache 214 if expr in cache: 215 return cache[expr] 216 217 pat = compile_uncached(expr, flags) 218 cache[expr] = pat 219 return pat 220 221 def icompile(expr): 222 return re_compile(expr, IGNORECASE) 223 224 def cond(*args): 225 if len(args) == 0: 226 return None 227 228 for i, e in enumerate(args): 229 if i % 2 == 0 and i < len(args) - 1 and e: 230 return args[i + 1] 231 232 return args[-1] if len(args) % 2 == 1 else None 233 234 def dive(into, using): 235 'Depth-first recursive caller for 1-input functions.' 236 237 if callable(into): 238 into, using = using, into 239 240 def rec(v): 241 if isinstance(v, dict): 242 return {k: rec(v) for k, v in v.items()} 243 if isinstance(v, Iterable) and not isinstance(v, str): 244 return [rec(v) for v in v] 245 return using(v) 246 247 return rec(into) 248 249 def divekeys(into, using): 250 'Depth-first recursive caller for 2-input funcs which rename dict keys.' 251 252 if callable(into): 253 into, using = using, into 254 255 def rec(v): 256 if isinstance(v, dict): 257 return {using(k): rec(v) for k, v in v.items()} 258 if isinstance(v, Iterable) and not isinstance(v, str): 259 return [rec(v) for i, v in enumerate(v)] 260 return v 261 262 return rec(None, into) 263 264 def divekv(into, using, using2 = None): 265 'Depth-first recursive caller for 2-input functions.' 266 267 if using2 is None: 268 if callable(into): 269 into, using = using, into 270 else: 271 if not callable(using2): 272 into, using, using2 = using2, into, using 273 274 def rec(k, v): 275 if isinstance(v, dict): 276 return {k: rec(k, v) for k, v in v.items()} 277 if isinstance(v, Iterable) and not isinstance(v, str): 278 return [rec(i, v) for i, v in enumerate(v)] 279 return using(k, v) 280 281 def rec2(k, v): 282 if isinstance(v, dict): 283 return {str(using(k, v)): rec2(k, v) for k, v in v.items()} 284 if isinstance(v, Iterable) and not isinstance(v, str): 285 return [rec2(i, v) for i, v in enumerate(v)] 286 return using2(k, v) 287 288 return rec(None, into) if using2 is None else rec2(None, into) 289 290 kvdive = divekv 291 292 def drop(src, *what): 293 if isinstance(src, str): 294 for s in what: 295 src = src.replace(s, '') 296 return src 297 298 def kdrop(src, what): 299 return {k: v for (k, v) in src.items() if not (k in what)} 300 301 if isinstance(src, dict): 302 return kdrop(src, set(what)) 303 304 if isinstance(src, Iterable): 305 what = set(what) 306 return [kdrop(e, what) for e in src if isinstance(e, dict)] 307 308 return None 309 310 dropped = drop 311 312 def join(x, *y): 313 'Join values into a string, or make a dict from keys and values.' 314 315 if len(y) == 0: 316 return ' '.join(str(v) for v in x) 317 if isinstance(x, str): 318 return x.join(str(v) for v in y) 319 if len(y) == 1 and isinstance(y[0], str): 320 return y[0].join(str(v) for v in x) 321 if len(y) == 1 and isinstance(y[0], (list, range, set, tuple, Generator)): 322 return {k: v for k, v in zip_longest(x, y[0]) if not (k is None)} 323 if isinstance(y, (list, range, set, tuple, Generator)): 324 return {k: v for k, v in zip_longest(x, y) if not (k is None)} 325 return {k: y for k in x} 326 327 def maybe(f, x): 328 try: 329 return f(x) 330 except Exception as _: 331 return x 332 333 def number(x): 334 try: 335 return int(x) 336 except Exception as _: 337 pass 338 try: 339 return float(x) 340 except Exception as _: 341 return x 342 343 def pick(src, *keys): 344 if isinstance(src, dict): 345 return {k: src.get(k, None) for k in keys} 346 return [{k: e.get(k, None) for k in keys} for e in src if isinstance(e, dict)] 347 348 picked = pick 349 350 def plain(s): 351 'Ignore all ANSI-style sequences in a string.' 352 return re_compile('''\x1b\\[([0-9;]+m|[0-9]*[A-HJKST])''').sub('', s) 353 354 def predicate(x): 355 'Helps various higher-order funcs, by standardizing `predicate` values.' 356 if callable(x): 357 return x 358 if isinstance(x, float): 359 if isnan(x): 360 return lambda y: isinstance(y, float) and isnan(y) 361 if isinf(x): 362 return lambda y: isinstance(y, float) and isinf(y) 363 return lambda y: x == y 364 365 def rescue(attempt, fallback = None): 366 try: 367 return attempt() 368 except Exception as e: 369 if callable(fallback): 370 return fallback(e) 371 return fallback 372 373 rescued = rescue 374 375 def retype(x): 376 'Try to narrow the type of the value given.' 377 378 if isinstance(x, float): 379 n = int(x) 380 return n if float(n) == x else x 381 382 if not isinstance(x, str): 383 return x 384 385 try: 386 return loads(x) 387 except Exception: 388 pass 389 390 try: 391 return int(x) 392 except Exception: 393 pass 394 395 try: 396 return float(x) 397 except Exception: 398 pass 399 400 return x 401 402 autocast = autocasted = mold = molded = recast = recasted = remold = retype 403 remolded = retyped = retype 404 405 def json0(x): 406 if isinstance(x, (range, set, Generator)): 407 x = tuple(x) 408 return dumps(x, separators=(',', ':'), allow_nan=False, indent=None) 409 410 j0 = json0 411 412 def typeof(x): 413 return { 414 type(None): 'null', 415 bool: 'boolean', 416 dict: 'object', 417 float: 'number', 418 int: 'number', 419 str: 'string', 420 list: 'array', 421 tuple: 'array', 422 }.get(type(x), 'other') 423 424 jstype = typeof 425 426 def after(x, what): 427 i = x.find(what) 428 return '' if i < 0 else x[i+len(what):] 429 430 def afterlast(x, what): 431 i = x.rfind(what) 432 return '' if i < 0 else x[i+len(what):] 433 434 afterfinal = afterlast 435 436 def before(x, what): 437 i = x.find(what) 438 return x if i < 0 else x[:i] 439 440 def beforelast(x, what): 441 i = x.rfind(what) 442 return x if i < 0 else x[:i] 443 444 beforefinal = beforelast 445 446 def since(x, what): 447 i = x.find(what) 448 return '' if i < 0 else x[i:] 449 450 def sincelast(x, what): 451 i = x.rfind(what) 452 return '' if i < 0 else x[i:] 453 454 sincefinal = sincelast 455 456 def until(x, what): 457 i = x.find(what) 458 return x if i < 0 else x[:i+len(what)] 459 460 def untilfinal(x, what): 461 i = x.rfind(what) 462 return x if i < 0 else x[:i+len(what)] 463 464 untillast = untilfinal 465 466 def message(msg, result = None): 467 print(msg, file=stderr) 468 return result 469 470 msg = message 471 472 # seen is used by func `once` to remember previously-given values 473 seen = set() 474 475 def once(x): 476 if x in seen: 477 return None 478 seen.add(x) 479 return x 480 481 dedup = unique = once 482 483 def utf8(x): 484 try: 485 if isinstance(x, str): 486 x = x.encode('utf-8') 487 return str(x, 'utf-8') 488 except Exception: 489 return None 490 491 def ymdhms(when = None): 492 fmt = f'%Y-%m-%d %H:%M:%S' 493 if isinstance(when, (float, int)): 494 return strftime(fmt, localtime(float(when))) 495 if isinstance(when, tuple): 496 return strftime(fmt, when) 497 return strftime(fmt, localtime()) 498 499 500 dquo = dquote = '"' 501 lcurly = '{' 502 rcurly = '}' 503 squo = squote = '\'' 504 505 nil = none = null = None 506 507 508 exec = None 509 open_utf8 = make_open_utf8(open) 510 open = open_utf8 511 512 json_opts = ('-j', '--j', '-json', '--json') 513 compact_json_opts = ( 514 '-c', '--c', '-compact', '--compact', 515 '-j0', '--j0', '-json0', '--json0', '-json-0', '--json-0', 516 ) 517 modules_opts = ( 518 '-m', '--m', '-mod', '--mod', '-module', '--module', 519 '-modules', '--modules', 520 ) 521 trace_opts = ('-t', '--t', '-trace', '--trace', '-traceback', '--traceback') 522 523 args = argv[1:] 524 if any(seemsurl(e) for e in args): 525 from io import TextIOWrapper 526 from urllib.request import urlopen 527 528 emit_json = False 529 compact_json = False 530 trace_errors = False 531 532 while len(args) > 0: 533 if args[0] == '--': 534 args = args[1:] 535 break 536 537 if args[0] in modules_opts: 538 try: 539 if len(args) < 2: 540 msg = 'a module name or a comma-separated list of modules' 541 raise Exception('expected ' + msg) 542 543 g = globals() 544 from importlib import import_module 545 for e in args[1].split(','): 546 g[e] = import_module(e) 547 548 g = None 549 import_module = None 550 args = args[2:] 551 except Exception as e: 552 fail(e, 1) 553 554 continue 555 556 if args[0] in json_opts: 557 emit_json = True 558 args = args[1:] 559 continue 560 561 if args[0] in compact_json_opts: 562 compact_json = True 563 args = args[1:] 564 continue 565 566 if args[0] in trace_opts: 567 trace_errors = True 568 args = args[1:] 569 continue 570 571 break 572 573 574 # ensure live-lines output, unless stdout is being saved into a file 575 flushed = stdout.isatty() or not stdout.seekable() 576 577 first_key = None 578 579 expr = '.' 580 if len(args) > 0: 581 expr = args[0] 582 args = args[1:] 583 584 if expr == '.': 585 expr = 'line' 586 587 try: 588 expr = compile(expr, expr, mode='eval') 589 compile = None 590 591 i = 0 592 n = 1 593 v = val = value = Skip() 594 595 groups = {} 596 items = fields = words = [] 597 tsv = [] 598 599 if len(args) == 0: 600 group_lines(stdin, expr) 601 602 if args.count('-') > 1: 603 msg = 'reading from `-` (standard input) more than once not allowed' 604 raise ValueError(msg) 605 606 for path in args: 607 if path == '-': 608 group_lines(stdin, expr) 609 continue 610 611 if seemsurl(path): 612 with urlopen(path) as inp: 613 with TextIOWrapper(inp, encoding='utf-8') as txt: 614 group_lines(txt, expr) 615 continue 616 617 with open_utf8(path) as txt: 618 group_lines(txt, expr) 619 620 if compact_json: 621 dump(groups, stdout, check_circular=False, allow_nan=False) 622 print() 623 elif emit_json: 624 dump(groups, stdout, check_circular=False, allow_nan=False, 625 separators=(', ', ': '), indent=4) 626 print() 627 else: 628 for i, group in enumerate(groups.values()): 629 # all items from the first key/group are already output 630 if i == 0: 631 continue 632 for e in group: 633 print(e) 634 except BrokenPipeError: 635 # quit quietly, instead of showing a confusing error message 636 stderr.close() 637 exit(0) 638 except KeyboardInterrupt: 639 exit(2) 640 except Exception as e: 641 if trace_errors: 642 raise e 643 else: 644 fail(e, 1)