File: tj.py 1 #!/usr/bin/python3 2 3 # The MIT License (MIT) 4 # 5 # Copyright © 2024 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the “Software”), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tj [options...] [python expression] [filepath/URI...] 28 29 30 Transform Json loads JSON data, runs a Python expression on it, and emits 31 the result as JSON. Parsed input-data are available to the expression as 32 any of the variables named `v`, `value`, `d`, and `data`. 33 34 If no file/URI is given, it loads JSON data from its standard input. If the 35 argument before the expression is a single equals sign (a `=`, without the 36 quotes), no data are read/parsed, and the expression is evaluated as given. 37 38 Options, where leading double-dashes are also allowed, except for alias `=`: 39 40 -c compact single-line JSON output (JSON-0) 41 -compact same as -c 42 -j0 same as -c 43 -json0 same as -c 44 -json-0 same as -c 45 46 -h show this help message 47 -help same as -h 48 49 -nil don't read any input 50 -no-input same as -nil 51 -noinput same as -nil 52 -none same as -nil 53 -null same as -nil 54 -null-input same as -nil 55 -nullinput same as -nil 56 = same as -nil 57 58 -d recursively make dictionary values dot-accessible 59 -dot same as -d 60 -dots same as -d 61 62 -p show a performance/time-profile of the expression run 63 -prof same as -p 64 -profile same as -p 65 66 -t show a full traceback of this script for exceptions 67 -trace same as -t 68 -traceback same as -t 69 70 -z zoom JSON value read from stdin, using all the keys given 71 -zj same as -z 72 -zoom same as -z 73 74 75 Extra Functions 76 77 after(x, y) ignore items until the one given; for strings and sequences 78 afterfinal(x, y) backward counterpart of func after 79 afterlast(x, y) same as func afterfinal 80 arrayish(x) check if value is a list, a tuple, or a generator 81 basename(s) get the final/file part of a pathname 82 before(x, y) ignore items since the one given; for strings and sequences 83 beforefinal(x, y) backward counterpart of func before 84 beforelast(x, y) same as func beforefinal 85 chunk(x, size) split/resequence items into chunks of the length given 86 chunked(x, size) same as func chunk 87 compose(*args) make a func which chain-calls all funcs given 88 composed(*args) same as func compose 89 cond(*args) expression-friendly fully-evaluated if-else chain 90 debase64(s) decode base64 strings, including data-URIs 91 dedup(x) ignore later (re)occurrences of values in a sequence 92 dejson(x, f=None) safe parse JSON from strings 93 denan(x, y) turn a floating-point NaN values into the fallback given 94 denil(*args) return the first non-null/none value among those given 95 denone(*args) same as func denil 96 denull(*args) same as func denil 97 dirname(s) get the folder/directory/parent part of a pathname 98 dive(x, f) transform value in depth-first-recursive fashion 99 divebin(x, y, f) binary (2-input) version of recursive-transform func dive 100 drop(x, *what) ignore keys or substrings; for strings, dicts, dict-lists 101 dropped(x, *v) same as func drop 102 each(x, f) generalization of built-in func map 103 endict(x) turn non-dictionary values into dicts with string keys 104 enfloat(x, f=nan) turn values into floats, offering a fallback on failure 105 enint(x, f=None) turn values into ints, offering a fallback on failure 106 enlist(x) turn non-list values into lists 107 entuple(x) turn non-tuple values into tuples 108 ext(s) return the file-extension part of a pathname, if available 109 fields(s) split fields AWK-style from the string given 110 filtered(x, f) same as func keep 111 flat(*args) flatten everything into an unnested sequence 112 fromto(x, y, ?f) sequence integers, end-value included 113 group(x, ?by) group values into dicts of lists; optional transform func 114 grouped(x, ?by) same as func group 115 harden(f, v) make funcs which return values instead of exceptions 116 hardened(f, v) same as func harden 117 countif(x, f) count how many values make the func given true-like 118 idiota(x, ?f) dict-counterpart of func iota 119 ints(x, y, ?f) make sequences of increasing integers, which include the end 120 iota(x, ?f) make an integer sequence from 1 up to the number given 121 join(x, y) join values into a string; make a dict from keys and values 122 json0(x) turn a value into its smallest JSON-string representation 123 json2(x) turn a value into a 2-space-indented multi-line JSON string 124 jsonl(x) turn a value into a sequence of single-line (JSONL) strings 125 keep(x, pred) generalization of built-in func filter 126 kept(x, pred) same as func keep 127 links(x) auto-detect all hyperlink-like (HTTP/HTTPS) substrings 128 mapped(x, f) same as func each 129 number(x) try to parse as an int, on failure try to parse as a float 130 numbers(x) auto-detect all numbers in the value given 131 numstats(x) calculate various `single-pass` numeric stats 132 once(x, y=None) avoid returning the same value more than once; stateful func 133 pick(x, *what) keep only the keys given; works on dicts, or dict-sequences 134 picked(x, *what) same a func pick 135 plain(s) ignore ANSI-style sequences in strings 136 quoted(s, q='"') surround a string with the (optional) quoting-symbol given 137 recover(*args) recover from exceptions with a fallback value 138 reject(x, pred) generalization of built-in func filter, with opposite logic 139 since(x, y) ignore items before the one given; for strings and sequences 140 sincefinal(x, y) backward counterpart of func since 141 sincelast(x, y) same as func sincefinal 142 split(x, y) split string by separator; split sequence into several ones 143 squeeze(s) strip/trim a string, squishing inner runs of spaces 144 stround(x, d=6) format numbers into decimal-number strings 145 tally(x, ?by) count/tally values, using an optional transformation func 146 tallied(x, ?by) same as func tally 147 trap(x, f=None) try running a func, handing exceptions to a fallback func 148 trycall(*args) same as func recover 149 unique(x) same as func dedup 150 uniqued(x) same as func dedup 151 unjson(x, f=None) same as func dejson 152 unquoted(s) ignore surrounding quotes, if present 153 until(x, y) ignore items after the one given; for strings and sequences 154 untilfinal(x, y) backward counterpart of func until 155 untillast(x, y) same as func untilfinal 156 wait(seconds, x) wait the given number of seconds, before returning a value 157 wat(*args) What Are These (wat) shows help/doc messages for funcs 158 159 160 Examples 161 162 # numbers from 0 to 5; no input is read/used 163 tj = 'range(6)' 164 165 # using bases 1 to 5, find all their powers up to the 4th 166 tj = '((n**p for p in range(1, 4+1)) for n in range(1, 6))' 167 168 # keep only the last 2 items from the input 169 tj = 'range(1, 6)' | tj 'data[-2:]' 170 171 # chunk/regroup input items into arrays of up to 3 items each 172 tj = 'range(1, 8)' | tj 'chunk(data, 3)' 173 174 # ignore all items before the first one with just a 5 in it 175 tj = 'range(8)' | tj 'since(data, 5)' 176 177 # ignore errors/exceptions, in favor of a fallback value 178 tj = 'safe(lambda: 2 * float("no way"), "fallback value")' 179 180 # ignore errors/exceptions, calling a fallback func with the exception 181 tj = 'safe(lambda: 2 * float("no way"), lambda err: str(err))' 182 183 # use dot-syntax on JSON data 184 tj = '{"abc": {"xyz": 123}}' | tj -d 'data.abc.xyz' 185 186 # use dot-syntax on JSON data; keywords as properties are syntax-errors 187 tj = '{"abc": {"def": 123}}' | tj -d 'data.abc["def"]' 188 189 # func results are automatically called on the input 190 tj = '{"abc": 123, "def": 456}' | tj len 191 ''' 192 193 194 from sys import argv, exit, stderr, stdin, stdout 195 196 197 if __name__ != '__main__': 198 print('don\'t import this script, run it directly instead', file=stderr) 199 exit(1) 200 201 # no args or a leading help-option arg means show the help message and quit 202 help_opts = ('-h', '--h', '-help', '--help') 203 if len(argv) < 2 or (len(argv) == 2 and argv[1] in help_opts): 204 print(info.strip(), file=stderr) 205 exit(0) 206 207 208 from io import StringIO 209 from itertools import islice 210 from json import load 211 212 from typing import \ 213 AbstractSet, Annotated, Any, AnyStr, \ 214 AsyncContextManager, AsyncGenerator, AsyncIterable, AsyncIterator, \ 215 Awaitable, BinaryIO, ByteString, Callable, cast, \ 216 ClassVar, Collection, Container, \ 217 ContextManager, Coroutine, Deque, Dict, Final, \ 218 final, ForwardRef, FrozenSet, Generator, Generic, get_args, get_origin, \ 219 get_type_hints, Hashable, IO, ItemsView, \ 220 Iterable, Iterator, KeysView, List, Literal, Mapping, \ 221 MappingView, Match, MutableMapping, MutableSequence, MutableSet, \ 222 NamedTuple, NewType, no_type_check, no_type_check_decorator, \ 223 NoReturn, Optional, overload, \ 224 Protocol, Reversible, \ 225 runtime_checkable, Sequence, Set, Sized, SupportsAbs, \ 226 SupportsBytes, SupportsComplex, SupportsFloat, SupportsIndex, \ 227 SupportsInt, SupportsRound, Text, TextIO, Tuple, Type, \ 228 TypedDict, TypeVar, \ 229 TYPE_CHECKING, Union, ValuesView 230 try: 231 from typing import \ 232 assert_never, assert_type, clear_overloads, Concatenate, \ 233 dataclass_transform, get_overloads, is_typeddict, LiteralString, \ 234 Never, NotRequired, ParamSpec, ParamSpecArgs, ParamSpecKwargs, \ 235 Required, reveal_type, Self, TypeAlias, TypeGuard, TypeVarTuple, \ 236 Unpack 237 from typing import \ 238 AwaitableGenerator, override, TypeAliasType, type_check_only 239 except Exception: 240 pass 241 242 243 def conforms(x: Any) -> bool: 244 ''' 245 Check if a value is JSON-compatible, which includes checking values 246 recursively, in case of composite/nestable values. 247 ''' 248 249 if x is None or isinstance(x, (bool, int, str)): 250 return True 251 if isinstance(x, float): 252 return not (isnan(x) or isinf(x)) 253 if isinstance(x, (list, tuple)): 254 return all(conforms(e) for e in x) 255 if isinstance(x, dict): 256 return all(conforms(k) and conforms(v) for k, v in x.items()) 257 return False 258 259 260 def seems_url(s: str) -> bool: 261 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 262 return any(s.startswith(p) for p in protocols) 263 264 265 def result_needs_fixing(x: Any) -> bool: 266 ''' 267 See if func fix_result needs to be called: avoiding that can speed 268 things up and save memory, when a composite value is big enough. 269 ''' 270 271 if x is None or isinstance(x, (bool, int, float, str)): 272 return False 273 rec = result_needs_fixing 274 if isinstance(x, dict): 275 return any(rec(k) or rec(v) for k, v in x.items()) 276 if isinstance(x, (list, tuple)): 277 return any(rec(e) for e in x) 278 return True 279 280 281 def fix_result(x: Any, default: Any) -> Any: 282 'Adapt a value so it can be output.' 283 284 if x is type: 285 return type(default).__name__ 286 287 # if expression results in a func, auto-call it with the original data 288 if callable(x): 289 c = required_arg_count(x) 290 if c == 1: 291 x = x(default) 292 else: 293 m = f'func auto-call only works with 1-arg funcs (func wanted {c})' 294 raise Exception(m) 295 296 if x is None or isinstance(x, (bool, int, float, str)): 297 return x 298 299 rec = fix_result 300 301 if isinstance(x, dict): 302 return { 303 rec(k, default): rec(v, default) for k, v in x.items() if not 304 (isinstance(k, Skip) or isinstance(v, Skip)) 305 } 306 if isinstance(x, Iterable): 307 return tuple(rec(e, default) for e in x if not isinstance(e, Skip)) 308 309 if isinstance(x, Dottable): 310 return rec(x.__dict__, default) 311 if isinstance(x, DotCallable): 312 return rec(x.value, default) 313 314 if isinstance(x, Exception): 315 raise x 316 317 return None if isinstance(x, Skip) else str(x) 318 319 320 def disabled_exec(*args, **kwargs) -> None: 321 _ = args 322 _ = kwargs 323 raise Exception('built-in func `exec` is disabled') 324 325 326 def disabled_open(*args, **kwargs) -> None: 327 _ = args 328 _ = kwargs 329 raise Exception('built-in func `open` is disabled') 330 331 332 from re import compile as zj_compile_re 333 334 335 zj_info_msg = ''' 336 zj [keys/indices...] 337 338 339 Zoom Json digs into a subset of valid JSON input, using the given mix of 340 keys and array-indices, the latter being either 0-based or negative, to 341 index backward from the ends of arrays. 342 343 Zooming on object keys is first tried as an exact key-match, failing that 344 as a case-insensitive key-match (first such match): when both approaches 345 fail, if the key is a valid integer, the key at the (even negative) index 346 given is used. 347 348 Invalid array-indices and missing object-keys result in null values, when 349 none of the special keys/fallbacks shown later apply. 350 351 You can slice arrays the exclusive/go/python way using index-pairs with a 352 `:` between the start/end pair, as long as it's a single argument; you can 353 even use `..` as the index-pair separator to include the stop index in the 354 result. Either way, as with go/python, you can omit either of the indices 355 when slicing. 356 357 Special key `.` acts as implicit loops on arrays, and even objects without 358 that specific key: in the unlikely case that an object has `.` as one of 359 its keys, you can use one of loop-fallback aliases, shown later. 360 361 Another special key is `+` (no quotes): when used, the rest of the keys 362 are used `in parallel`, allowing multiple picks from the current value. 363 When picking array items, you can also use either type (`:` or `..`) of 364 slicing, even mixing it with individual indices. 365 366 Similar to `+`, the `-` fallback-key drops keys, which means all items are 367 picked, except for those mentioned after the `-`. 368 369 Unlike the looping special key, after the first `+` special-key, all keys 370 following it, special or not, are picked normally. 371 372 In case any of the special keys are actual keys in the data loaded, some 373 aliases are available: 374 375 . /. ./ :. .: 376 + /+ +/ :+ +: 377 - /- -/ :- -: 378 379 .i :i .info :info :info: 380 .k :k .keys :keys :keys: 381 .t :t .type :type :type: 382 .l :l .len .length :len :len: :length :length: 383 384 These aliases allow using the special functionality even on objects whose 385 keys match some of these special names, as it's extremely unlikely data use 386 all aliases as actual keys at any level. 387 388 The only input supported is valid JSON coming from standard-input: there's 389 no way to load files using their names. To load data from files/URIs use 390 tools like `cat` or `curl`, and pipe their output into this tool. 391 ''' 392 393 zj_slice_re = zj_compile_re('''^(([+-]?[0-9]+)?)(:|\.\.)(([+-]?[0-9]+)?)$''') 394 395 396 def zj_zoom(data: Any, keys: Tuple[str, ...]) -> Any: 397 eval_due = False 398 399 for i, k in enumerate(keys): 400 try: 401 if eval_due: 402 data = eval(k)(data) 403 eval_due = False 404 continue 405 406 if isinstance(data, dict): 407 m = zj_match_key(data, k) 408 if m in data: 409 data = data[m] 410 continue 411 m = zj_slice_re.match(k) 412 if m: 413 data = {k: data[k] for k in zj_match_keys(data, k)} 414 continue 415 416 if isinstance(data, (list, tuple)): 417 try: 418 i = int(k) 419 l = len(data) 420 data = data[i] if -l <= i < l else None 421 continue 422 except Exception: 423 m = zj_slice_re.match(k) 424 if m: 425 data = [data[i] for i in zj_match_indices(data, k)] 426 continue 427 428 if k in ('.pyl', ':pyl', 'pyl:', ':pyl:'): 429 eval_due = True 430 continue 431 432 if k in ('.', '/.', './', ':.', '.:'): 433 if isinstance(data, dict): 434 rest = tuple(keys[i + 1:]) 435 return {k: zj_zoom(v, rest) for k, v in data.items()} 436 if isinstance(data, (list, tuple)): 437 rest = tuple(keys[i + 1:]) 438 return tuple(zj_zoom(v, rest) for v in data) 439 440 # doing nothing amounts to an identity-op for simple values 441 continue 442 443 fn = zj_final_fallbacks.get(k, None) 444 if fn: 445 return fn(data, tuple(keys[i + 1:])) 446 447 fn = zj_fallbacks.get(k, None) 448 if fn: 449 data = fn(data) 450 continue 451 452 if isinstance(data, (dict, list, tuple)): 453 data = None 454 continue 455 456 kind = zj_type(data) 457 msg = f'value of type {kind} has no properties to zoom into' 458 raise Exception(msg) 459 except Exception as e: 460 key_path = ' > '.join(islice(keys, None, i + 1)) 461 raise Exception(f'{key_path}: {e}') 462 463 return data 464 465 466 def zj_match_key(src: Dict, key: str) -> str: 467 if key in src: 468 return key 469 470 low = key.casefold() 471 for k in src.keys(): 472 if low == k.casefold(): 473 return k 474 475 try: 476 i = int(key) 477 l = len(src) 478 if i < 0: 479 i += l 480 if i < 0 or i >= l: 481 return None 482 for j, k in enumerate(src.keys()): 483 if i == j: 484 return k 485 except Exception: 486 return key 487 return key 488 489 490 def zj_match_keys(src: Any, key: str) -> Iterable: 491 if isinstance(src, (list, tuple)): 492 yield from zj_match_indices(src, key) 493 yield from zj_match_fallbacks(src, key) 494 return 495 496 if isinstance(src, dict): 497 if key in src: 498 yield key 499 return 500 501 low = key.casefold() 502 for k in src.keys(): 503 if low == k.casefold(): 504 yield k 505 return 506 507 yield from zj_match_indices(src, key) 508 yield from zj_match_fallbacks(src, key) 509 return 510 511 yield from zj_match_fallbacks(src, key) 512 513 514 def zj_match_indices(src: Any, key: str) -> Iterable: 515 try: 516 i = int(key) 517 518 if isinstance(src, (list, tuple)): 519 l = len(src) 520 yield src[i] if -l <= i < l else None 521 return 522 523 if isinstance(src, dict): 524 l = len(src) 525 if i < 0: 526 i += l 527 if i < 0 or i >= l: 528 return 529 530 for j, k in enumerate(src.keys()): 531 if i == j: 532 yield k 533 return 534 535 return 536 except Exception: 537 pass 538 539 m = zj_slice_re.match(key) 540 if not m: 541 return 542 543 l = len(src) 544 545 (start, _, kind, stop, _) = m.groups() 546 start = int(start) if start != '' else 0 547 stop = int(stop) if stop != '' else l 548 549 if start < 0: 550 start += l 551 start = max(start, 0) 552 if stop < 0: 553 stop += l 554 stop = min(stop, l) 555 if kind == '..': 556 stop += 1 557 stop = min(stop, l) 558 559 if start > stop: 560 return 561 if (start < 0 and stop < 0) or (start >= l and stop >= l): 562 return 563 564 565 if isinstance(src, dict): 566 for i, k in enumerate(src.keys()): 567 if i >= stop: 568 return 569 if start <= i: 570 yield k 571 return 572 573 if isinstance(src, (list, tuple)): 574 yield from range(start, stop) 575 return 576 577 578 579 def zj_match_fallbacks(src: Any, key: str) -> Iterable: 580 fn = zj_fallbacks.get(key, None) 581 if fn: 582 yield fn(src) 583 584 585 def zj_help(*_) -> NoReturn: 586 print(zj_info_msg.strip(), file=stderr) 587 exit(1) 588 589 590 def zj_keys(src: Any) -> Any: 591 if isinstance(src, dict): 592 return tuple(src.keys()) 593 if isinstance(src, (list, tuple)): 594 return tuple(range(len(src))) 595 return None 596 597 598 def zj_info(x: Any) -> str: 599 if isinstance(x, dict): 600 return f'object ({len(x)} items)' 601 if isinstance(x, (list, tuple)): 602 return f'array ({len(x)} items)' 603 return zj_type(x) 604 605 606 def zj_type(x: Any) -> str: 607 return { 608 type(None): 'null', 609 dict: 'object', 610 float: 'number', 611 int: 'number', 612 str: 'string', 613 list: 'array', 614 tuple: 'array', 615 }.get(type(x), 'other') 616 617 618 zj_fallbacks: Dict[str, Callable] = { 619 '.h': zj_help, 620 '.help': zj_help, 621 ':h': zj_help, 622 ':help': zj_help, 623 ':help:': zj_help, 624 '.i': zj_info, 625 '.info': zj_info, 626 ':i': zj_info, 627 ':info': zj_info, 628 ':info:': zj_info, 629 '.k': zj_keys, 630 '.keys': zj_keys, 631 ':keys': zj_keys, 632 ':keys:': zj_keys, 633 '.l': len, 634 '.len': len, 635 '.length': len, 636 ':l': len, 637 ':len': len, 638 ':length': len, 639 ':len:': len, 640 ':length:': len, 641 '.t': zj_type, 642 '.type': zj_type, 643 ':t': zj_type, 644 ':type': zj_type, 645 ':type:': zj_type, 646 } 647 648 649 def zj_pick(src: Any, keys: Tuple[str, ...]) -> Any: 650 if isinstance(src, dict): 651 picked = {} 652 for k in keys: 653 for k in zj_match_keys(src, k): 654 picked[k] = src[k] 655 return picked 656 657 if isinstance(src, (list, tuple)): 658 picked = [] 659 for k in keys: 660 for i in zj_match_indices(src, k): 661 picked.append(src[i]) 662 return tuple(picked) 663 664 msg = f'can\'t pick properties from value of type {zj_type(src)}' 665 raise Exception(msg) 666 667 668 def zj_drop(src: Any, keys: Tuple[str, ...]) -> Any: 669 if isinstance(src, dict): 670 avoid = set() 671 for k in keys: 672 for k in zj_match_keys(src, k): 673 avoid.add(k) 674 return {k: v for k, v in src.items() if not k in avoid} 675 676 if isinstance(src, (list, tuple)): 677 l = len(src) 678 avoid = set() 679 for k in keys: 680 for i in zj_match_indices(src, k): 681 avoid.add(i if i >= 0 else i + l) 682 return tuple(v for i, v in enumerate(src) if not i in avoid) 683 684 msg = f'can\'t drop properties from value of type {zj_type(src)}' 685 raise Exception(msg) 686 687 688 zj_final_fallbacks: Dict[str, Callable] = { 689 '+': zj_pick, 690 ':+:': zj_pick, 691 ':+': zj_pick, 692 '+:': zj_pick, 693 '/+': zj_pick, 694 '+/': zj_pick, 695 696 '-': zj_drop, 697 ':-:': zj_drop, 698 ':-': zj_drop, 699 '-:': zj_drop, 700 '/-': zj_drop, 701 '-/': zj_drop, 702 } 703 704 705 from base64 import \ 706 standard_b64encode, standard_b64decode, \ 707 standard_b64encode as base64bytes, standard_b64decode as debase64bytes 708 709 from collections import \ 710 ChainMap, Counter, defaultdict, deque, namedtuple, OrderedDict, \ 711 UserDict, UserList, UserString 712 713 from copy import copy, deepcopy 714 715 from datetime import \ 716 MAXYEAR, MINYEAR, date, datetime, time, timedelta, timezone, tzinfo 717 try: 718 from datetime import now, UTC 719 except Exception: 720 now = lambda: datetime(2000, 1, 1).now() 721 722 from decimal import Decimal, getcontext 723 724 from difflib import \ 725 context_diff, diff_bytes, Differ, get_close_matches, HtmlDiff, \ 726 IS_CHARACTER_JUNK, IS_LINE_JUNK, ndiff, restore, SequenceMatcher, \ 727 unified_diff 728 729 from fractions import Fraction 730 731 import functools 732 from functools import \ 733 cache, cached_property, cmp_to_key, get_cache_token, lru_cache, \ 734 namedtuple, partial, partialmethod, recursive_repr, reduce, \ 735 singledispatch, singledispatchmethod, total_ordering, update_wrapper, \ 736 wraps 737 738 from glob import glob, iglob 739 740 try: 741 from graphlib import CycleError, TopologicalSorter 742 except Exception: 743 pass 744 745 from hashlib import \ 746 file_digest, md5, pbkdf2_hmac, scrypt, sha1, sha224, sha256, sha384, \ 747 sha512 748 749 from inspect import getfullargspec, getsource 750 751 import itertools 752 from itertools import \ 753 accumulate, chain, combinations, combinations_with_replacement, \ 754 compress, count, cycle, dropwhile, filterfalse, groupby, islice, \ 755 permutations, product, repeat, starmap, takewhile, tee, zip_longest 756 try: 757 from itertools import pairwise 758 from itertools import batched 759 except Exception: 760 pass 761 762 from json import dump, dumps, loads 763 764 import math 765 Math = math 766 from math import \ 767 acos, acosh, asin, asinh, atan, atan2, atanh, ceil, comb, \ 768 copysign, cos, cosh, degrees, dist, e, erf, erfc, exp, expm1, \ 769 fabs, factorial, floor, fmod, frexp, fsum, gamma, gcd, hypot, inf, \ 770 isclose, isfinite, isinf, isnan, isqrt, lcm, ldexp, lgamma, log, \ 771 log10, log1p, log2, modf, nan, nextafter, perm, pi, pow, prod, \ 772 radians, remainder, sin, sinh, sqrt, tan, tanh, tau, trunc, ulp 773 try: 774 from math import cbrt, exp2 775 except Exception: 776 pass 777 778 power = pow 779 780 import operator 781 782 from pathlib import Path 783 784 from pprint import \ 785 isreadable, isrecursive, pformat, pp, pprint, PrettyPrinter, saferepr 786 787 from random import \ 788 betavariate, choice, choices, expovariate, gammavariate, gauss, \ 789 getrandbits, getstate, lognormvariate, normalvariate, paretovariate, \ 790 randbytes, randint, random, randrange, sample, seed, setstate, \ 791 shuffle, triangular, uniform, vonmisesvariate, weibullvariate 792 793 compile_py = compile # keep built-in func compile for later 794 from re import compile as compile_uncached, Pattern, IGNORECASE 795 796 import statistics 797 from statistics import \ 798 bisect_left, bisect_right, fmean, \ 799 geometric_mean, harmonic_mean, mean, median, \ 800 median_grouped, median_high, median_low, mode, multimode, pstdev, \ 801 pvariance, quantiles, stdev, variance 802 try: 803 from statistics import \ 804 correlation, covariance, linear_regression, mul 805 except Exception: 806 pass 807 808 import string 809 from string import \ 810 Formatter, Template, ascii_letters, ascii_lowercase, ascii_uppercase, \ 811 capwords, digits, hexdigits, octdigits, printable, punctuation, \ 812 whitespace 813 814 alphabet = ascii_letters 815 letters = ascii_letters 816 lowercase = ascii_lowercase 817 uppercase = ascii_uppercase 818 819 from textwrap import dedent, fill, indent, shorten, wrap 820 821 from time import \ 822 altzone, asctime, \ 823 ctime, daylight, get_clock_info, \ 824 gmtime, localtime, mktime, monotonic, monotonic_ns, perf_counter, \ 825 perf_counter_ns, process_time, process_time_ns, \ 826 sleep, strftime, strptime, struct_time, thread_time, thread_time_ns, \ 827 time, time_ns, timezone, tzname 828 try: 829 from time import \ 830 clock_getres, clock_gettime, clock_gettime_ns, clock_settime, \ 831 clock_settime_ns, pthread_getcpuclockid, tzset 832 except Exception: 833 pass 834 835 from unicodedata import \ 836 bidirectional, category, combining, decimal, decomposition, digit, \ 837 east_asian_width, is_normalized, lookup, mirrored, name, normalize, \ 838 numeric 839 840 from urllib.parse import \ 841 parse_qs, parse_qsl, quote, quote_from_bytes, quote_plus, unquote, \ 842 unquote_plus, unquote_to_bytes, unwrap, urldefrag, urlencode, urljoin, \ 843 urlparse, urlsplit, urlunparse, urlunsplit 844 845 846 class Skip: 847 'Custom type which some funcs type-check to skip values in containers.' 848 849 def __init__(self, *args) -> None: 850 pass 851 852 853 # skip is a ready-to-use value which some funcs filter against: this way 854 # filtering values becomes a special case of transforming values 855 skip = Skip() 856 857 # re_cache is used by custom func compile to cache previously-compiled 858 # regular-expressions, which makes them quicker to (re)use in formulas 859 re_cache: Dict[str, Pattern] = {} 860 861 # ire_cache is like re_cache, except it's for case-insensitive regexes 862 ire_cache: Dict[str, Pattern] = {} 863 864 # ansi_style_re detects the most commonly-used ANSI-style sequences, and 865 # is used in func plain 866 ansi_style_re = compile_uncached('\x1b\\[([0-9;]+m|[0-9]*[A-HJKST])') 867 868 # number_re detects numbers, and is used in func numbers 869 number_re = compile_uncached('\\W-?[0-9]+(\\.[0-9]*)?\\W') 870 871 # link_re detects web links, and is used in func links 872 link_re_src = 'https?://[A-Za-z0-9+_.:%-]+(/[A-Za-z0-9+_.%/,#?&=-]*)*' 873 link_re = compile_uncached(link_re_src) 874 875 # paddable_tab_re detects single tabs and possible runs of spaces around 876 # them, and is used in func squeeze 877 paddable_tab_re = compile_uncached(' *\t *') 878 879 # seen remembers values already given to func `once` 880 seen = set() 881 882 # commented_re detects strings/lines which start as unix-style comments 883 commented_re = compile_uncached('^ *#') 884 885 # emptyish_re detects empty/emptyish strings/lines, the latter being strings 886 # with only spaces in them 887 emptyish_re = compile_uncached('^ *\r?\n?$') 888 889 # spaces_re detects runs of 2 or more spaces, and is used in func squeeze 890 spaces_re = compile_uncached(' +') 891 892 # awk_sep_re splits like AWK does by default, and is used in func fields 893 awk_sep_re = compile_uncached(' *\t *| +') 894 895 896 # some convenience aliases to commonly-used values 897 898 false = False 899 true = True 900 nil = None 901 nihil = None 902 none = None 903 null = None 904 s = '' 905 906 months = [ 907 'January', 'February', 'March', 'April', 'May', 'June', 908 'July', 'August', 'September', 'October', 'November', 'December', 909 ] 910 911 monweek = [ 912 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 913 'Saturday', 'Sunday', 914 ] 915 916 sunweek = [ 917 'Sunday', 918 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 919 ] 920 921 phy = { 922 'kilo': 1_000, 923 'mega': 1_000_000, 924 'giga': 1_000_000_000, 925 'tera': 1_000_000_000_000, 926 'peta': 1_000_000_000_000_000, 927 'exa': 1_000_000_000_000_000_000, 928 'zetta': 1_000_000_000_000_000_000_000, 929 930 'c': 299_792_458, 931 'kcd': 683, 932 'na': 602214076000000000000000, 933 934 'femto': 1e-15, 935 'pico': 1e-12, 936 'nano': 1e-9, 937 'micro': 1e-6, 938 'milli': 1e-3, 939 940 'e': 1.602176634e-19, 941 'f': 96_485.33212, 942 'h': 6.62607015e-34, 943 'k': 1.380649e-23, 944 'mu': 1.66053906892e-27, 945 946 'ge': 9.7803267715, 947 'gn': 9.80665, 948 } 949 950 physics = phy 951 952 # using literal strings on the cmd-line is often tricky/annoying: some of 953 # these aliases can help get around multiple levels of string-quoting; no 954 # quotes are needed as the script will later make these values accessible 955 # via the property/dot syntax 956 sym = { 957 'amp': '&', 958 'ampersand': '&', 959 'ansiclear': '\x1b[0m', 960 'ansinormal': '\x1b[0m', 961 'ansireset': '\x1b[0m', 962 'apo': '\'', 963 'apos': '\'', 964 'ast': '*', 965 'asterisk': '*', 966 'at': '@', 967 'backquote': '`', 968 'backslash': '\\', 969 'backtick': '`', 970 'ball': '●', 971 'bang': '!', 972 'bigsigma': 'Σ', 973 'block': '█', 974 'bquo': '`', 975 'bquote': '`', 976 'bslash': '\\', 977 'btick': '`', 978 'bullet': '•', 979 'caret': '^', 980 'cdot': '·', 981 'circle': '●', 982 'colon': ':', 983 'comma': ',', 984 'cr': '\r', 985 'crlf': '\r\n', 986 'cross': '×', 987 'cs': ', ', 988 'dash': '—', 989 'dollar': '$', 990 'dot': '.', 991 'dquo': '"', 992 'dquote': '"', 993 'emark': '!', 994 'emdash': '—', 995 'empty': '', 996 'endash': '–', 997 'eq': '=', 998 'et': '&', 999 'euro': '€', 1000 'ge': '≥', 1001 'geq': '≥', 1002 'gt': '>', 1003 'hellip': '…', 1004 'hole': '○', 1005 'hyphen': '-', 1006 'infinity': '∞', 1007 'lcurly': '{', 1008 'ldquo': '“', 1009 'ldquote': '“', 1010 'le': '≤', 1011 'leq': '≤', 1012 'lf': '\n', 1013 'lt': '<', 1014 'mdash': '—', 1015 'mdot': '·', 1016 'miniball': '•', 1017 'minus': '-', 1018 'ndash': '–', 1019 'neq': '≠', 1020 'perc': '%', 1021 'percent': '%', 1022 'period': '.', 1023 'plus': '+', 1024 'qmark': '?', 1025 'que': '?', 1026 'rcurly': '}', 1027 'rdquo': '”', 1028 'rdquote': '”', 1029 'sball': '•', 1030 'semi': ';', 1031 'semicolon': ';', 1032 'sharp': '#', 1033 'slash': '/', 1034 'space': ' ', 1035 'square': '■', 1036 'squo': '\'', 1037 'squote': '\'', 1038 'tab': '\t', 1039 'tilde': '~', 1040 'underscore': '_', 1041 'uscore': '_', 1042 'utf8bom': '\xef\xbb\xbf', 1043 'utf16be': '\xfe\xff', 1044 'utf16le': '\xff\xfe', 1045 } 1046 1047 symbols = sym 1048 1049 units = { 1050 'cup2l': 0.23658824, 1051 'floz2l': 0.0295735295625, 1052 'floz2ml': 29.5735295625, 1053 'ft2m': 0.3048, 1054 'gal2l': 3.785411784, 1055 'in2cm': 2.54, 1056 'lb2kg': 0.45359237, 1057 'mi2km': 1.609344, 1058 'mpg2kpl': 0.425143707, 1059 'nmi2km': 1.852, 1060 'oz2g': 28.34952312, 1061 'psi2pa': 6894.757293168, 1062 'ton2kg': 907.18474, 1063 'yd2m': 0.9144, 1064 1065 'mol': 602214076000000000000000, 1066 'mole': 602214076000000000000000, 1067 1068 'hour': 3_600, 1069 'day': 86_400, 1070 'week': 604_800, 1071 1072 'hr': 3_600, 1073 'wk': 604_800, 1074 1075 'kb': 1024, 1076 'mb': 1024**2, 1077 'gb': 1024**3, 1078 'tb': 1024**4, 1079 'pb': 1024**5, 1080 } 1081 1082 # some convenience aliases to various funcs from the python stdlib 1083 geomean = geometric_mean 1084 harmean = harmonic_mean 1085 sd = stdev 1086 popsd = pstdev 1087 var = variance 1088 popvar = pvariance 1089 randbeta = betavariate 1090 randexp = expovariate 1091 randgamma = gammavariate 1092 randlognorm = lognormvariate 1093 randnorm = normalvariate 1094 randweibull = weibullvariate 1095 1096 capitalize = str.capitalize 1097 casefold = str.casefold 1098 center = str.center 1099 # count = str.count 1100 decode = bytes.decode 1101 encode = str.encode 1102 endswith = str.endswith 1103 expandtabs = str.expandtabs 1104 find = str.find 1105 format = str.format 1106 index = str.index 1107 isalnum = str.isalnum 1108 isalpha = str.isalpha 1109 isascii = str.isascii 1110 isdecimal = str.isdecimal 1111 isdigit = str.isdigit 1112 isidentifier = str.isidentifier 1113 islower = str.islower 1114 isnumeric = str.isnumeric 1115 isprintable = str.isprintable 1116 isspace = str.isspace 1117 istitle = str.istitle 1118 isupper = str.isupper 1119 # join = str.join 1120 ljust = str.ljust 1121 lower = str.lower 1122 lowered = str.lower 1123 lstrip = str.lstrip 1124 maketrans = str.maketrans 1125 partition = str.partition 1126 removeprefix = str.removeprefix 1127 removesuffix = str.removesuffix 1128 replace = str.replace 1129 rfind = str.rfind 1130 rindex = str.rindex 1131 rjust = str.rjust 1132 rpartition = str.rpartition 1133 rsplit = str.rsplit 1134 rstrip = str.rstrip 1135 # split = str.split 1136 splitlines = str.splitlines 1137 startswith = str.startswith 1138 strip = str.strip 1139 swapcase = str.swapcase 1140 title = str.title 1141 translate = str.translate 1142 upper = str.upper 1143 uppered = str.upper 1144 zfill = str.zfill 1145 1146 every = all 1147 rev = reversed 1148 reverse = reversed 1149 some = any 1150 1151 length = len 1152 1153 blowtabs = str.expandtabs 1154 hasprefix = str.startswith 1155 hassuffix = str.endswith 1156 ltrim = str.lstrip 1157 stripstart = str.lstrip 1158 trimspace = str.strip 1159 trimstart = str.lstrip 1160 rtrim = str.rstrip 1161 stripend = str.rstrip 1162 trimend = str.rstrip 1163 stripped = str.strip 1164 trim = str.strip 1165 trimmed = str.strip 1166 trimprefix = str.removeprefix 1167 trimsuffix = str.removesuffix 1168 1169 1170 def required_arg_count(f: Callable) -> int: 1171 if isinstance(f, type): 1172 return 1 1173 1174 meta = getfullargspec(f) 1175 n = len(meta.args) 1176 if meta.defaults: 1177 n -= len(meta.defaults) 1178 return n 1179 1180 1181 def identity(x: Any) -> Any: 1182 ''' 1183 Return the value given: this is the default transformer for several 1184 higher-order funcs, which effectively keeps original items as given. 1185 ''' 1186 return x 1187 1188 idem = identity 1189 iden = identity 1190 1191 1192 def after(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1193 'Skip parts of strings/sequences up to the substring/value given.' 1194 return (strafter if isinstance(x, str) else itemsafter)(x, what) 1195 1196 def afterlast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1197 'Skip parts of strings/sequences up to the last substring/value given.' 1198 return (strafterlast if isinstance(x, str) else itemsafterlast)(x, what) 1199 1200 afterfinal = afterlast 1201 1202 def arrayish(x: Any) -> bool: 1203 'Check if a value is array-like enough.' 1204 return isinstance(x, (list, tuple, range, Generator)) 1205 1206 isarrayish = arrayish 1207 1208 def base64(x): 1209 return base64bytes(str(x).encode()).decode() 1210 1211 def basename(s: str) -> str: 1212 'Get a filepath\'s last part, if present.' 1213 return Path(s).name 1214 1215 def before(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1216 'End strings/sequences right before a substring/value\'s appearance.' 1217 return (strbefore if isinstance(x, str) else itemsbefore)(x, what) 1218 1219 def beforelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1220 'End strings/sequences right before a substring/value\'s last appearance.' 1221 return (strbeforelast if isinstance(x, str) else itemsbeforelast)(x, what) 1222 1223 beforefinal = beforelast 1224 1225 def cases(x: Any, *args: Any) -> Any: 1226 ''' 1227 Simulate a switch statement on a value, using matches/result pairs from 1228 the arguments given; when given an even number of extra args, None is 1229 used as a final fallback result; when given an odd number of extra args, 1230 the last argument is used as a final `default` value, if needed. 1231 ''' 1232 1233 for i in range(0, len(args) - len(args) % 2, 2): 1234 test, res = args[i], args[i+1] 1235 if isinstance(test, (list, tuple)) and x in test: 1236 return res 1237 if isinstance(test, float) and isnan(test) and isnan(x): 1238 return res 1239 if x == test: 1240 return res 1241 return None if len(args) % 2 == 0 else args[-1] 1242 1243 switch = cases 1244 1245 def chunk(items: Iterable, chunk_size: int) -> Iterable: 1246 'Break iterable into chunks, each with up to the item-count given.' 1247 1248 if isinstance(items, str): 1249 n = len(items) 1250 while n >= chunk_size: 1251 yield items[:chunk_size] 1252 items = items[chunk_size:] 1253 n -= chunk_size 1254 if n > 0: 1255 yield items 1256 return 1257 1258 if not isinstance(chunk_size, int): 1259 raise Exception('non-integer chunk-size') 1260 if chunk_size < 1: 1261 raise Exception('non-positive chunk-size') 1262 1263 it = iter(items) 1264 while True: 1265 head = tuple(islice(it, chunk_size)) 1266 if not head: 1267 return 1268 yield head 1269 1270 chunked = chunk 1271 1272 def commented(s: str) -> bool: 1273 'Check if a string starts as a unix-style comment.' 1274 return commented_re.match(s) != None 1275 1276 iscommented = commented 1277 1278 def compile(s: str, case_sensitive: bool = True) -> Pattern: 1279 'Cached regex `compiler`, so it\'s quicker to (re)use in formulas.' 1280 1281 cache = re_cache if case_sensitive else ire_cache 1282 options = 0 if case_sensitive else IGNORECASE 1283 1284 if s in cache: 1285 return cache[s] 1286 e = compile_uncached(s, options) 1287 cache[s] = e 1288 return e 1289 1290 def compose(*what: Callable) -> Callable: 1291 def composite(x: Any) -> Any: 1292 for f in what: 1293 x = f(x) 1294 return x 1295 return composite 1296 1297 composed = compose 1298 lcompose = compose 1299 lcomposed = compose 1300 1301 def cond(*args: Any) -> Any: 1302 ''' 1303 Simulate a chain of if-else statements, using condition/result pairs 1304 from the arguments given; when given an even number of args, None is 1305 used as a final fallback result; when given an odd number of args, the 1306 last argument is used as a final `else` value, if needed. 1307 ''' 1308 1309 for i in range(0, len(args) - len(args) % 2, 2): 1310 if args[i]: 1311 return args[i+1] 1312 return None if len(args) % 2 == 0 else args[-1] 1313 1314 def conform(x: Any, denan: Any = None, deinf: Any = None, fn = str) -> Any: 1315 'Make values JSON-compatible.' 1316 1317 if isinstance(x, float): 1318 # turn NaNs and Infinities into the replacement values given 1319 if isnan(x): 1320 return denan 1321 if isinf(x): 1322 return deinf 1323 return x 1324 1325 if isinstance(x, (bool, int, float, str)): 1326 return x 1327 1328 if isinstance(x, dict): 1329 return { 1330 str(k): conform(v) for k, v in x.items() if not 1331 (isinstance(k, Skip) or isinstance(v, Skip)) 1332 } 1333 1334 if isinstance(x, Iterable): 1335 return [conform(e) for e in x if not isinstance(e, Skip)] 1336 1337 if isinstance(x, DotCallable): 1338 return x.value 1339 1340 return fn(x) 1341 1342 fix = conform 1343 1344 def countif(src: Iterable, check: Callable) -> int: 1345 ''' 1346 Count how many values make the func given true-like. This func works with 1347 sequences, dictionaries, and strings. 1348 ''' 1349 1350 if callable(src): 1351 src, check = check, src 1352 check = predicate(check) 1353 1354 total = 0 1355 if isinstance(src, dict): 1356 for v in src.values(): 1357 if check(v): 1358 total += 1 1359 else: 1360 for v in src: 1361 if check(v): 1362 total += 1 1363 return total 1364 1365 # def debase64(x): 1366 # return debase64bytes(str(x).encode()).decode() 1367 1368 def debase64(s: str) -> bytes: 1369 'Convert away from base64 encoding, including data-URIs.' 1370 1371 if s.startswith('data:'): 1372 i = s.find(',') 1373 if i >= 0: 1374 return standard_b64decode(s[i + 1:]) 1375 return standard_b64decode(s) 1376 1377 unbase64 = debase64 1378 1379 def dedup(v: Iterable) -> Iterable: 1380 'Ignore reappearing items from iterables, after their first occurrence.' 1381 1382 got = set() 1383 for e in v: 1384 if not e in got: 1385 got.add(e) 1386 yield e 1387 1388 dedupe = dedup 1389 deduped = dedup 1390 deduplicate = dedup 1391 deduplicated = dedup 1392 undup = dedup 1393 undupe = dedup 1394 unduped = dedup 1395 unduplicate = dedup 1396 unduplicated = dedup 1397 unique = dedup 1398 uniqued = dedup 1399 1400 def defunc(x: Any) -> Any: 1401 'Call if value is a func, or return it back as given.' 1402 return x() if callable(x) else x 1403 1404 callmemaybe = defunc 1405 defunct = defunc 1406 unfunc = defunc 1407 unfunct = defunc 1408 1409 def dejson(x: Any, catch: Union[Callable[[Exception], Any], Any] = None) -> Any: 1410 'Safely parse JSON from strings.' 1411 try: 1412 return loads(x) if isinstance(x, str) else x 1413 except Exception as e: 1414 return catch(e) if callable(catch) else catch 1415 1416 unjson = dejson 1417 1418 def denan(x: Any, fallback: Any = None) -> Any: 1419 'Replace floating-point NaN with the alternative value given.' 1420 return x if not (isinstance(x, float) and isnan(x)) else fallback 1421 1422 def denil(*args: Any) -> Any: 1423 'Avoid None values, if possible: first value which isn\'t None wins.' 1424 for e in args: 1425 if e != None: 1426 return e 1427 return None 1428 1429 denone = denil 1430 denull = denil 1431 1432 def dirname(s: str) -> str: 1433 'Ignore the last part of a filepath.' 1434 return str(Path(s).parent) 1435 1436 def dive(into: Any, doing: Callable) -> Any: 1437 'Transform a nested value by calling a func via depth-first recursion.' 1438 1439 # support args in either order 1440 if callable(into): 1441 into, doing = doing, into 1442 1443 return _dive_kv(None, into, doing) 1444 1445 deepmap = dive 1446 dive1 = dive 1447 1448 def divebin(x: Any, y: Any, doing: Callable) -> Any: 1449 'Nested 2-value version of depth-first-recursive func dive.' 1450 1451 # support args in either order 1452 if callable(x): 1453 x, y, doing = y, doing, x 1454 1455 narg = required_arg_count(doing) 1456 if narg == 2: 1457 return dive(x, lambda a: dive(y, lambda b: doing(a, b))) 1458 if narg == 4: 1459 return dive(x, lambda i, a: dive(y, lambda j, b: doing(i, a, j, b))) 1460 raise Exception('divebin(...) only supports funcs with 2 or 4 args') 1461 1462 bindive = divebin 1463 # diveboth = divebin 1464 # dualdive = divebin 1465 # duodive = divebin 1466 dive2 = divebin 1467 1468 def _dive_kv(key: Any, into: Any, doing: Callable) -> Any: 1469 if isinstance(into, dict): 1470 return {k: _dive_kv(k, v, doing) for k, v in into.items()} 1471 if isinstance(into, Iterable) and not isinstance(into, str): 1472 return [_dive_kv(i, e, doing) for i, e in enumerate(into)] 1473 1474 narg = required_arg_count(doing) 1475 return doing(key, into) if narg == 2 else doing(into) 1476 1477 class DotCallable: 1478 'Enable convenient dot-syntax calling of 1-input funcs.' 1479 1480 def __init__(self, value: Any): 1481 self.value = value 1482 1483 def __getattr__(self, key: str) -> Any: 1484 return DotCallable(globals()[key](self.value)) 1485 1486 class Dottable: 1487 'Enable convenient dot-syntax access to dictionary values.' 1488 1489 def __getattr__(self, key: Any) -> Any: 1490 return self.__dict__[key] if key in self.__dict__ else None 1491 1492 def __getitem__(self, key: Any) -> Any: 1493 return self.__dict__[key] if key in self.__dict__ else None 1494 1495 def __iter__(self) -> Iterable: 1496 return iter(self.__dict__) 1497 1498 def dotate(x: Any) -> Union[Dottable, Any]: 1499 'Recursively ensure all dictionaries in a value are dot-accessible.' 1500 1501 if isinstance(x, dict): 1502 d = Dottable() 1503 d.__dict__ = {k: dotate(v) for k, v in x.items()} 1504 return d 1505 if isinstance(x, list): 1506 return [dotate(e) for e in x] 1507 if isinstance(x, tuple): 1508 return tuple(dotate(e) for e in x) 1509 return x 1510 1511 dotated = dotate 1512 dote = dotate 1513 doted = dotate 1514 dotified = dotate 1515 dotify = dotate 1516 dottified = dotate 1517 dottify = dotate 1518 1519 # make dictionaries `physics`, `symbols`, and `units` easier to use 1520 phy = dotate(phy) 1521 physics = phy 1522 sym = dotate(sym) 1523 symbols = sym 1524 units = dotate(units) 1525 1526 def drop(src: Any, *what) -> Any: 1527 ''' 1528 Either ignore all substrings occurrences, or ignore all keys given from 1529 an object, or even from a sequence of objects. 1530 ''' 1531 1532 if isinstance(src, str): 1533 return strdrop(src, *what) 1534 return _itemsdrop(src, set(what)) 1535 1536 dropped = drop 1537 # ignore = drop 1538 # ignored = drop 1539 1540 def _itemsdrop(src: Any, what: Set) -> Any: 1541 if isinstance(src, dict): 1542 kv = {} 1543 for k, v in src.items(): 1544 if not (k in what): 1545 kv[k] = v 1546 return kv 1547 1548 if isinstance(src, Iterable): 1549 return [_itemsdrop(e, what) for e in src] 1550 1551 return None 1552 1553 def each(src: Iterable, f: Callable) -> Any: 1554 ''' 1555 A generalization of built-in func map, which can also handle dictionaries 1556 and strings. 1557 ''' 1558 1559 if callable(src): 1560 src, f = f, src 1561 1562 if isinstance(src, dict): 1563 return mapkv(src, lambda k, _: k, f) 1564 1565 if isinstance(src, str): 1566 s = StringIO() 1567 f = loopify(f) 1568 for i, c in enumerate(src): 1569 v = f(i, c) 1570 if not isinstance(v, Skip): 1571 s.write(str(v)) 1572 return s.getvalue() 1573 1574 return tuple(f(i, v) for i, v in enumerate(src)) 1575 1576 mapped = each 1577 1578 def emptyish(x: Any) -> bool: 1579 ''' 1580 Check if a value can be considered empty, which includes non-empty 1581 strings which only have spaces in them. 1582 ''' 1583 1584 def check(x: Any) -> bool: 1585 if not x: 1586 return True 1587 if isinstance(x, str): 1588 return bool(emptyish_re.match(x)) 1589 return False 1590 1591 if check(x): 1592 return True 1593 if isinstance(x, Iterable): 1594 return all(check(e) for e in x) 1595 return False 1596 1597 isemptyish = emptyish 1598 1599 def endict(x: Any) -> Dict[str, Any]: 1600 'Turn non-dictionary values into dictionaries with string keys.' 1601 1602 if isinstance(x, dict): 1603 return {str(k): v for k, v in x.items()} 1604 if arrayish(x): 1605 return {str(e): e for e in x} 1606 return {str(x): x} 1607 1608 dicted = endict 1609 endicted = endict 1610 indict = endict 1611 todict = endict 1612 1613 def enfloat(x: Any, fallback: float = nan) -> float: 1614 try: 1615 return float(x) 1616 except Exception: 1617 return fallback 1618 1619 enfloated = enfloat 1620 floated = enfloat 1621 floatify = enfloat 1622 floatize = enfloat 1623 tofloat = enfloat 1624 1625 def enint(x: Any, fallback: Any = None) -> Any: 1626 try: 1627 return int(x) 1628 except Exception: 1629 return fallback 1630 1631 eninted = enint 1632 inted = enint 1633 integered = enint 1634 intify = enint 1635 intize = enint 1636 toint = enint 1637 1638 def enlist(x: Any) -> List[Any]: 1639 'Turn non-list values into lists.' 1640 return list(x) if arrayish(x) else [x] 1641 1642 # inlist = enlist 1643 enlisted = enlist 1644 listify = enlist 1645 listize = enlist 1646 tolist = enlist 1647 1648 def entuple(x: Any) -> Tuple[Any, ...]: 1649 'Turn non-tuple values into tuples.' 1650 return tuple(x) if arrayish(x) else (x, ) 1651 1652 entupled = entuple 1653 ntuple = entuple 1654 ntupled = entuple 1655 tuplify = entuple 1656 tuplize = entuple 1657 toentuple = entuple 1658 tontuple = entuple 1659 totuple = entuple 1660 1661 def error(message: Any) -> Exception: 1662 return Exception(str(message)) 1663 1664 err = error 1665 1666 def ext(s: str) -> str: 1667 'Get a filepath\'s extension, if present.' 1668 1669 name = Path(s).name 1670 i = name.rfind('.') 1671 return name[i:] if i >= 0 else '' 1672 1673 filext = ext 1674 1675 def fail(message: Any, error_code: int = 255) -> NoReturn: 1676 stdout.flush() 1677 print(f'\x1b[31m{message}\x1b[0m', file=stderr) 1678 quit(error_code) 1679 1680 abort = fail 1681 bail = fail 1682 1683 def fields(s: str) -> Iterable[str]: 1684 'Split fields AWK-style from the string given.' 1685 return awk_sep_re.split(s.strip()) 1686 1687 # items = fields 1688 splitfields = fields 1689 splititems = fields 1690 words = fields 1691 1692 def first(items: SupportsIndex, fallback: Any = None) -> Any: 1693 return items[0] if len(items) > 0 else fallback 1694 1695 def flappend(*args: Any) -> List[Any]: 1696 'Turn arbitrarily-nested values/sequences into a single flat sequence.' 1697 1698 flat = [] 1699 def dig(x: Any) -> None: 1700 if arrayish(x): 1701 for e in x: 1702 dig(e) 1703 elif isinstance(x, dict): 1704 for e in x.values(): 1705 dig(e) 1706 else: 1707 flat.append(x) 1708 1709 for e in args: 1710 dig(e) 1711 return flat 1712 1713 def flat(*args: Any) -> Iterable: 1714 'Turn arbitrarily-nested values/sequences into a single flat sequence.' 1715 1716 def _flat_rec(x: Any) -> Iterable: 1717 if x is None: 1718 return 1719 1720 if isinstance(x, dict): 1721 yield from _flat_rec(x.values()) 1722 1723 if isinstance(x, str): 1724 yield x 1725 return 1726 1727 if isinstance(x, Iterable): 1728 for e in x: 1729 yield from _flat_rec(e) 1730 return 1731 1732 yield x 1733 1734 for x in args: 1735 yield from _flat_rec(x) 1736 1737 flatten = flat 1738 flattened = flat 1739 1740 def fromto(start, stop, f: Callable = identity) -> Iterable: 1741 'Sequence all integers between the numbers given, end-value included.' 1742 return (f(e) for e in range(start, stop + 1)) 1743 1744 def fuzz(x: Union[int, float]) -> Union[float, Dict[str, float]]: 1745 ''' 1746 Deapproximate numbers to their max range before approximation: the 1747 result is a dictionary with the guessed lower-bound number, the number 1748 given, and the guessed upper-bound number which can approximate to the 1749 original number given. NaNs and the infinities are returned as given, 1750 instead of resulting in a dictionary. 1751 ''' 1752 1753 if isnan(x) or isinf(x): 1754 return x 1755 1756 if x == 0: 1757 return {'-0.5': -0.5, '0': 0.0, '0.5': +0.5} 1758 1759 if x % 1 != 0: 1760 # return surrounding integers when given non-integers 1761 a = floor(x) 1762 b = ceil(x) 1763 return {str(a): a, str(x): x, str(b): b} 1764 1765 if x % 10 != 0: 1766 a = x - 0.5 1767 b = x + 0.5 1768 return {str(a): a, str(x): x, str(b): b} 1769 1770 # find the integer log10 of the absolute value; 0 was handled previously 1771 y = int(abs(x)) 1772 p10 = 1 1773 while True: 1774 if y % p10 != 0: 1775 p10 /= 10 1776 break 1777 p10 *= 10 1778 delta = p10 / 2 1779 1780 s = +1 if x > 0 else -1 1781 ux = abs(x) 1782 a = s * ux - delta 1783 b = s * ux + delta 1784 return {str(a): a, str(x): x, str(b): b} 1785 1786 def generated(src: Any) -> Any: 1787 'Make tuples out of generators, or return non-generator values as given.' 1788 return tuple(src) if isinstance(src, (Generator, range)) else src 1789 1790 concrete = generated 1791 concreted = generated 1792 concretize = generated 1793 concretized = generated 1794 degen = generated 1795 degenerate = generated 1796 degenerated = generated 1797 degenerator = generated 1798 gen = generated 1799 generate = generated 1800 synth = generated 1801 synthed = generated 1802 synthesize = generated 1803 synthesized = generated 1804 1805 def group(src: Iterable, by: Callable = identity) -> Dict: 1806 ''' 1807 Separate transformed items into arrays, the final result being a dict 1808 whose keys are all the transformed values, and whose values are lists 1809 of all the original values which did transform to their group's key. 1810 ''' 1811 1812 if callable(src): 1813 src, by = by, src 1814 1815 by = loopify(by) 1816 kv = src.items() if isinstance(src, dict) else enumerate(src) 1817 1818 groups = {} 1819 for k, v in kv: 1820 dk = by(k, v) 1821 if isinstance(dk, Skip) or isinstance(v, Skip): 1822 continue 1823 if dk in groups: 1824 groups[dk].append(v) 1825 else: 1826 groups[dk] = [v] 1827 return groups 1828 1829 grouped = group 1830 1831 def gire(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict: 1832 ''' 1833 Group matched items into arrays, the final result being a dict whose 1834 keys are all the matchable regexes given, and whose values are lists 1835 of all the original values which did case-insensitively match their 1836 group's key as a regex. 1837 ''' 1838 1839 using = tuple(using) 1840 return group(src, lambda x: imatch(x, using, fallback)) 1841 1842 gbire = gire 1843 groupire = gire 1844 1845 def gre(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict: 1846 ''' 1847 Group matched items into arrays, the final result being a dict whose 1848 keys are all the matchable regexes given, and whose values are lists 1849 of all the original values which did regex-match their group's key. 1850 ''' 1851 1852 using = tuple(using) 1853 return group(src, lambda x: match(x, using, fallback)) 1854 1855 gbre = gre 1856 groupre = gre 1857 1858 def gsub(s: str, what: str, repl: str) -> str: 1859 'Replace all regex-matches with the string given.' 1860 return compile(what).sub(repl, s) 1861 1862 def harden(f: Callable, fallback: Any = None) -> Callable: 1863 def _hardened_caller(*args): 1864 try: 1865 return f(*args) 1866 except Exception: 1867 return fallback 1868 return _hardened_caller 1869 1870 hardened = harden 1871 insure = harden 1872 insured = harden 1873 1874 def horner(coeffs: List[float], x: Union[int, float]) -> float: 1875 if isinstance(coeffs, (int, float)): 1876 coeffs, x = x, coeffs 1877 1878 if len(coeffs) == 0: 1879 return 0 1880 1881 y = coeffs[0] 1882 for c in islice(coeffs, 1, None): 1883 y *= x 1884 y += c 1885 return y 1886 1887 polyval = horner 1888 1889 def idiota(n: int, f: Callable = identity) -> Dict[int, int]: 1890 'ID (keys) version of func iota.' 1891 return { v: v for v in (f(e) for e in range(1, n + 1))} 1892 1893 dictiota = idiota 1894 kviota = idiota 1895 1896 def imatch(what: str, using: Iterable[str], fallback: str = '') -> str: 1897 'Try to case-insensitively match a string with any of the regexes given.' 1898 1899 if not isinstance(what, str): 1900 what, using = using, what 1901 1902 for s in using: 1903 expr = compile(s, False) 1904 m = expr.search(what) 1905 if m: 1906 # return what[m.start():m.end()] 1907 return s 1908 return fallback 1909 1910 def indices(x: Any) -> Iterable[Any]: 1911 'List all indices/keys, or get an exclusive range from an int.' 1912 1913 if isinstance(x, int): 1914 return range(x) 1915 if isinstance(x, dict): 1916 return x.keys() 1917 if isinstance(x, (str, list, tuple)): 1918 return range(len(x)) 1919 return tuple() 1920 1921 keys = indices 1922 1923 def ints(start, stop, f: Callable = identity) -> Iterable[int]: 1924 'Sequence integers, end-value included.' 1925 1926 if isnan(start) or isnan(stop) or isinf(start) or isinf(stop): 1927 return tuple() 1928 return (f(e) for e in range(int(ceil(start)), int(stop) + 1)) 1929 1930 integers = ints 1931 1932 def iota(n: int, f: Callable = identity) -> Iterable[int]: 1933 'Sequence all integers from 1 up to (and including) the int given.' 1934 return (f(e) for e in range(1, n + 1)) 1935 1936 def itemsafter(x: Iterable, what: Any) -> Iterable: 1937 ok = False 1938 check = predicate(what) 1939 for e in x: 1940 if ok: 1941 yield e 1942 elif check(e): 1943 ok = True 1944 1945 def itemsafterlast(x: Iterable, what: Any) -> Iterable: 1946 rest: List[Any] = [] 1947 check = predicate(what) 1948 for e in x: 1949 if check(e): 1950 rest.clear() 1951 else: 1952 rest.append(e) 1953 1954 for e in islice(rest, 1, len(rest)): 1955 yield e 1956 1957 def itemsbefore(x: Iterable, what: Any) -> Iterable: 1958 check = predicate(what) 1959 for e in x: 1960 if check(e): 1961 return 1962 yield e 1963 1964 def itemsbeforelast(x: Iterable, what: Any) -> Iterable: 1965 items = [] 1966 for e in x: 1967 items.append(e) 1968 1969 i = -1 1970 check = predicate(what) 1971 for j, e in enumerate(reversed(items)): 1972 if check(e): 1973 i = j 1974 break 1975 1976 if i < 0: 1977 return items 1978 if i == 0: 1979 return tuple() 1980 for e in islice(items, 0, i): 1981 yield e 1982 1983 def itemssince(x: Iterable, what: Any) -> Iterable: 1984 ok = False 1985 check = predicate(what) 1986 for e in x: 1987 ok = ok or check(e) 1988 if ok: 1989 yield e 1990 1991 def itemssincelast(x: Iterable, what: Any) -> Iterable: 1992 rest: List[Any] = [] 1993 check = predicate(what) 1994 for e in x: 1995 if check(e): 1996 rest.clear() 1997 else: 1998 rest.append(e) 1999 return rest 2000 2001 def itemsuntil(x: Iterable, what: Any) -> Iterable: 2002 check = predicate(what) 2003 for e in x: 2004 yield e 2005 if check(e): 2006 return 2007 2008 def itemsuntillast(x: Iterable, what: Any) -> Iterable: 2009 items = [] 2010 for e in x: 2011 items.append(e) 2012 2013 i = -1 2014 check = predicate(what) 2015 for j, e in enumerate(reversed(items)): 2016 if check(e): 2017 i = j 2018 break 2019 2020 if i < 0: 2021 return items 2022 for e in islice(items, 0, i + 1): 2023 yield e 2024 2025 itemsuntilfinal = itemsuntillast 2026 2027 def join(items: Iterable, sep: Union[str, Iterable] = ' ') -> Union[str, Dict]: 2028 ''' 2029 Join iterables using the separator-string given: its 2 arguments 2030 can come in either order, and are sorted out if needed. When given 2031 2 non-string iterables, the result is an object whose keys are from 2032 the first argument, and whose values are from the second one. 2033 2034 You can use it any of the following ways, where `keys` and `values` are 2035 sequences (lists, tuples, or generators), and `separator` is a string: 2036 2037 join(values) 2038 join(values, separator) 2039 join(separator, values) 2040 join(keys, values) 2041 ''' 2042 2043 if arrayish(items) and arrayish(sep): 2044 return {k: v for k, v in zip(items, sep)} 2045 if isinstance(items, str): 2046 items, sep = sep, items 2047 return sep.join(str(e) for e in items) 2048 2049 def joined_paragraphs(lines: Iterable[str]) -> Iterable[Sequence[str]]: 2050 ''' 2051 Regroup lines into individual paragraphs, each of which can span multiple 2052 lines: such paragraphs have no empty lines in them, and never end with a 2053 trailing line-feed. 2054 ''' 2055 2056 par: List[str] = [] 2057 for l in lines: 2058 if (not l) and par: 2059 yield '\n'.join(par) 2060 par.clear() 2061 else: 2062 par.append(l) 2063 2064 if len(par) > 0: 2065 yield '\n'.join(par) 2066 2067 def json0(x: Any) -> str: 2068 'Encode value into a minimal single-line JSON string.' 2069 return dumps(x, separators=(',', ':'), allow_nan=False, indent=None) 2070 2071 j0 = json0 2072 2073 def json2(x: Any) -> str: 2074 ''' 2075 Encode value into a (possibly multiline) JSON string, using 2 spaces for 2076 each indentation level. 2077 ''' 2078 return dumps(x, separators=(',', ': '), allow_nan=False, indent=2) 2079 2080 j2 = json2 2081 2082 def jsonl(x: Any) -> Iterable: 2083 'Turn value into multiple JSON-encoded strings, known as JSON Lines.' 2084 2085 if x is None: 2086 yield dumps(x, allow_nan=False) 2087 elif isinstance(x, (bool, int, float, dict, str)): 2088 yield dumps(x, allow_nan=False) 2089 elif isinstance(x, Iterable): 2090 for e in x: 2091 yield dumps(e, allow_nan=False) 2092 else: 2093 yield dumps(str(x), allow_nan=False) 2094 2095 jsonlines = jsonl 2096 tojsonl = jsonl 2097 tojsonlines = jsonl 2098 2099 def keep(src: Iterable, pred: Any) -> Iterable: 2100 ''' 2101 A generalization of built-in func filter, which can also handle dicts and 2102 strings. 2103 ''' 2104 2105 if callable(src): 2106 src, pred = pred, src 2107 pred = predicate(pred) 2108 pred = loopify(pred) 2109 2110 if isinstance(src, str): 2111 out = StringIO() 2112 for i, c in enumerate(src): 2113 if pred(i, c): 2114 out.write(c) 2115 return out.getvalue() 2116 2117 if isinstance(src, dict): 2118 return { k: v for k, v in src.items() if pred(k, v) } 2119 return (e for i, e in enumerate(src) if pred(i, e)) 2120 2121 filtered = keep 2122 kept = keep 2123 2124 def last(items: SupportsIndex, fallback: Any = None) -> Any: 2125 return items[-1] if len(items) > 0 else fallback 2126 2127 def links(src: Any) -> Iterable: 2128 'Auto-detect all (HTTP/HTTPS) hyperlink-like substrings.' 2129 2130 if isinstance(src, str): 2131 for match in link_re.finditer(src): 2132 # yield src[match.start():match.end()] 2133 yield match.group(0) 2134 elif isinstance(src, dict): 2135 for k, v in src.items(): 2136 yield from k 2137 yield from links(v) 2138 elif isinstance(src, Iterable): 2139 for v in src: 2140 yield from links(v) 2141 2142 def loopify(x: Callable) -> Callable: 2143 nargs = required_arg_count(x) 2144 if nargs == 2: 2145 return x 2146 elif nargs == 1: 2147 return lambda _, v: x(v) 2148 else: 2149 raise Exception('only funcs with 1 or 2 args are supported') 2150 2151 def mapkv(src: Iterable, key: Callable, value: Callable = identity) -> Dict: 2152 ''' 2153 A map-like func for dictionaries, which uses 2 mapping funcs, the first 2154 for the keys, the second for the values. 2155 ''' 2156 2157 if key is None: 2158 key = lambda k, _: k 2159 2160 if callable(src): 2161 src, key, value = value, src, key 2162 2163 if required_arg_count(key) != 2: 2164 oldkey = key 2165 key = lambda k, _: oldkey(k) 2166 2167 key = loopify(key) 2168 value = loopify(value) 2169 # if isinstance(src, dict): 2170 # return { key(k, v): value(k, v) for k, v in src.items() } 2171 # return { key(i, v): value(i, v) for i, v in enumerate(src) } 2172 2173 def add(k, v, to): 2174 dk = key(k, v) 2175 dv = value(k, v) 2176 if isinstance(dk, Skip) or isinstance(dv, Skip): 2177 return 2178 to[dk] = dv 2179 2180 res = {} 2181 kv = src.items() if isinstance(src, dict) else enumerate(src) 2182 for k, v in kv: 2183 add(k, v, res) 2184 return res 2185 2186 def match(what: str, using: Iterable[str], fallback: str = '') -> str: 2187 'Try to match a string with any of the regexes given.' 2188 2189 if not isinstance(what, str): 2190 what, using = using, what 2191 2192 for s in using: 2193 expr = compile(s) 2194 m = expr.search(what) 2195 if m: 2196 # return what[m.start():m.end()] 2197 return s 2198 return fallback 2199 2200 def maybe(f: Callable, x: Any) -> Any: 2201 ''' 2202 Try calling a func on a value, using the same value as a fallback result, 2203 in case of exceptions. 2204 ''' 2205 2206 if not callable(f): 2207 f, x = x, f 2208 try: 2209 return f(x) 2210 except Exception: 2211 return x 2212 2213 def mappend(*args) -> Dict: 2214 kv = {} 2215 for src in args: 2216 if isinstance(src, dict): 2217 for k, v in src.items(): 2218 kv[k] = v 2219 else: 2220 raise Exception('mappend only works with dictionaries') 2221 return kv 2222 2223 def message(x: Any, result: Any = skip) -> Any: 2224 print(x, file=stderr) 2225 return result 2226 2227 msg = message 2228 2229 def must(cond: Any, errmsg: str = 'condition given not always true') -> None: 2230 'Enforce conditions, raising an exception on failure.' 2231 if not cond: 2232 raise Exception(errmsg) 2233 2234 demand = must 2235 enforce = must 2236 2237 def nowdict() -> dict: 2238 v = datetime(2000, 1, 1).now() 2239 return { 2240 'year': v.year, 2241 'month': v.month, 2242 'day': v.day, 2243 'hour': v.hour, 2244 'minute': v.minute, 2245 'second': v.second, 2246 'text': v.strftime('%Y-%m-%d %H:%M:%S %b %a'), 2247 'weekday': v.strftime('%A'), 2248 } 2249 2250 def number(x: Any) -> Union[int, float, Any]: 2251 ''' 2252 Try to turn the value given into a number, using a fallback value instead 2253 of raising exceptions. 2254 ''' 2255 2256 if isinstance(x, float): 2257 return x 2258 2259 try: 2260 return int(x) 2261 except Exception: 2262 return float(x) 2263 2264 def numbers(src: Any) -> Iterable: 2265 'Auto-detect all number-like substrings.' 2266 2267 if isinstance(src, str): 2268 for match in number_re.finditer(src): 2269 yield match.group(0) 2270 # yield src[match.start():match.end()] 2271 elif isinstance(src, dict): 2272 for k, v in src.items(): 2273 yield from k 2274 yield from links(v) 2275 elif isinstance(src, Iterable): 2276 for v in src: 2277 yield from links(v) 2278 2279 def numsign(x: Union[int, float]) -> Union[int, float]: 2280 'Get a number\'s sign, or NaN if the number given is a NaN.' 2281 2282 if isinstance(x, int): 2283 if x > 0: 2284 return +1 2285 if x < 0: 2286 return -1 2287 return 0 2288 2289 if isnan(x): 2290 return x 2291 2292 if x > 0: 2293 return +1.0 2294 if x < 0: 2295 return -1.0 2296 return 0.0 2297 2298 def numstats(src: Any) -> Dict[str, Union[float, int]]: 2299 'Gather several single-pass numeric statistics.' 2300 2301 n = mean_sq = ln_sum = 0 2302 least = +inf 2303 most = -inf 2304 total = mean = 0 2305 prod = 1 2306 nans = ints = pos = zero = neg = 0 2307 2308 def update_numstats(x: Any) -> None: 2309 nonlocal nans, n, ints, pos, neg, zero, least, most, total, prod 2310 nonlocal ln_sum, mean, mean_sq 2311 2312 if not isinstance(x, (float, int)): 2313 return 2314 2315 if isnan(x): 2316 nans += 1 2317 return 2318 2319 n += 1 2320 ints += int(isinstance(x, int) or x == floor(x)) 2321 2322 if x > 0: 2323 pos += 1 2324 elif x < 0: 2325 neg += 1 2326 else: 2327 zero += 1 2328 2329 least = min(least, x) 2330 most = max(most, x) 2331 2332 # total += x 2333 prod *= x 2334 ln_sum += log(x) 2335 2336 d1 = x - mean 2337 mean += d1 / n 2338 d2 = x - mean 2339 mean_sq += d1 * d2 2340 2341 def _numstats_rec(src: Any) -> None: 2342 if isinstance(src, dict): 2343 for e in src.values(): 2344 _numstats_rec(e) 2345 elif isinstance(src, Iterable) and not isinstance(src, str): 2346 for e in src: 2347 _numstats_rec(e) 2348 else: 2349 update_numstats(src) 2350 2351 _numstats_rec(src) 2352 2353 sd = nan 2354 geomean = nan 2355 if n > 0: 2356 sd = sqrt(mean_sq / n) 2357 geomean = exp(ln_sum / n) if not isinf(ln_sum) else nan 2358 total = n * mean 2359 2360 return { 2361 'n': n, 2362 'nan': nans, 2363 'min': least, 2364 'max': most, 2365 'sum': total, 2366 'mean': mean, 2367 'geomean': geomean, 2368 'sd': sd, 2369 'product': prod, 2370 'integer': ints, 2371 'positive': pos, 2372 'zero': zero, 2373 'negative': neg, 2374 } 2375 2376 def once(x: Any, replacement: Any = None) -> Any: 2377 ''' 2378 Replace the first argument given after the first time this func has been 2379 given it: this is a deliberately stateful function, given its purpose. 2380 ''' 2381 2382 if not (x in seen): 2383 seen.add(x) 2384 return x 2385 else: 2386 return replacement 2387 2388 onced = once 2389 2390 def pad(s: str, n: int, pad: str = ' ') -> str: 2391 l = len(s) 2392 return s if l >= n else s + int((n - l) / len(pad)) * pad 2393 2394 def padcenter(s: str, n: int, pad: str = ' ') -> str: 2395 return s.center(n, pad) 2396 2397 centerpad = padcenter 2398 centerpadded = padcenter 2399 cjust = padcenter 2400 cpad = padcenter 2401 padc = padcenter 2402 paddedcenter = padcenter 2403 2404 def padend(s: str, n: int, pad: str = ' ') -> str: 2405 return s.rjust(n, pad) 2406 2407 padr = padend 2408 padright = padend 2409 paddedend = padend 2410 paddedright = padend 2411 rpad = padend 2412 rightpad = padend 2413 rightpadded = padend 2414 2415 def padstart(s: str, n: int, pad: str = ' ') -> str: 2416 return s.ljust(n, pad) 2417 2418 lpad = padstart 2419 leftpad = padstart 2420 leftpadded = padstart 2421 padl = padstart 2422 padleft = padstart 2423 paddedleft = padstart 2424 paddedstart = padstart 2425 2426 def panic(x: Any) -> None: 2427 raise Exception(x) 2428 2429 def paragraphize(lines: Iterable[str]) -> Iterable[Sequence[str]]: 2430 ''' 2431 Regroup lines into individual paragraphs, each of which is a list of 2432 single-line strings, none of which never end with a trailing line-feed. 2433 ''' 2434 2435 par: List[str] = [] 2436 for l in lines: 2437 if (not l) and par: 2438 yield par 2439 par.clear() 2440 else: 2441 par.append(l) 2442 2443 if len(par) > 0: 2444 yield par 2445 2446 paragraphed = paragraphize 2447 paragraphs = paragraphize 2448 paragroup = paragraphize 2449 pargroup = paragraphize 2450 2451 def parse(s: str, fallback: Any = None) -> Any: 2452 'Try to parse JSON, ignoring exceptions in favor of a fallback value.' 2453 2454 try: 2455 return loads(s) 2456 except Exception: 2457 return fallback 2458 2459 fromjson = parse 2460 parsed = parse 2461 loaded = parse 2462 unjson = parse 2463 2464 def pick(src: Any, *what) -> Any: 2465 'Pick only the keys given from an object, or even a sequence of objects.' 2466 2467 if isinstance(src, dict): 2468 kv = {} 2469 for k in what: 2470 kv[k] = src[k] 2471 return kv 2472 2473 if isinstance(src, Iterable): 2474 return [pick(e, *what) for e in src] 2475 2476 return None 2477 2478 picked = pick 2479 2480 def plain(s: str) -> str: 2481 'Ignore all ANSI-style sequences in a string.' 2482 return ansi_style_re.sub('', s) 2483 2484 def predicate(x: Any) -> Callable: 2485 'Helps various higher-order funcs, by standardizing `predicate` values.' 2486 2487 if callable(x): 2488 return x 2489 2490 if isinstance(x, float): 2491 if isnan(x): 2492 return lambda y: isinstance(y, float) and isnan(y) 2493 if isinf(x): 2494 return lambda y: isinstance(y, float) and isinf(y) 2495 2496 return lambda y: x == y 2497 2498 pred = predicate 2499 2500 def quoted(s: str, quote: str = '"') -> str: 2501 'Surround a string with quotes.' 2502 return f'{quote}{s}{quote}' 2503 2504 def recover(*args) -> Any: 2505 ''' 2506 Catch exceptions using a lambda/callback func, in one of 6 ways 2507 recover(zero_args_func) 2508 recover(zero_args_func, exception_replacement_value) 2509 recover(zero_args_func, one_arg_exception_handling_func) 2510 recover(one_arg_func, arg) 2511 recover(one_arg_func, arg, exception_replacement_value) 2512 recover(one_arg_func, arg, one_arg_exception_handling_func) 2513 ''' 2514 2515 if len(args) == 1: 2516 f = args[0] 2517 try: 2518 return f() 2519 except Exception: 2520 return None 2521 elif len(args) == 2: 2522 f, fallback = args[0], args[1] 2523 if callable(f) and callable(fallback): 2524 try: 2525 return f() 2526 except Exception as e: 2527 nargs = required_arg_count(fallback) 2528 return fallback(e) if nargs == 1 else fallback() 2529 else: 2530 try: 2531 return f() if required_arg_count(f) == 0 else f(args[1]) 2532 except Exception: 2533 return fallback 2534 elif len(args) == 3: 2535 f, x, fallback = args[0], args[1], args[2] 2536 if callable(f) and callable(fallback): 2537 try: 2538 return f(x) 2539 except Exception as e: 2540 nargs = required_arg_count(fallback) 2541 return fallback(e) if nargs == 1 else fallback() 2542 else: 2543 try: 2544 return f(x) 2545 except Exception: 2546 return fallback 2547 else: 2548 raise Exception('recover(...) only works with 1, 2, or 3 args') 2549 2550 attempt = recover 2551 attempted = recover 2552 recovered = recover 2553 recoverred = recover 2554 rescue = recover 2555 rescued = recover 2556 trycall = recover 2557 2558 def reject(src: Iterable, pred: Any) -> Iterable: 2559 ''' 2560 A generalization of built-in func filter, which uses predicate funcs the 2561 opposite way, and which can also handle dicts and strings. 2562 ''' 2563 2564 if callable(src): 2565 src, pred = pred, src 2566 pred = predicate(pred) 2567 pred = loopify(pred) 2568 2569 if isinstance(src, str): 2570 out = StringIO() 2571 for i, c in enumerate(src): 2572 if not pred(i, c): 2573 out.write(c) 2574 return out.getvalue() 2575 2576 if isinstance(src, dict): 2577 return { k: v for k, v in src.items() if not pred(k, v) } 2578 return (e for i, e in enumerate(src) if not pred(i, e)) 2579 2580 avoid = reject 2581 avoided = reject 2582 keepout = reject 2583 keptout = reject 2584 rejected = reject 2585 2586 def retype(x: Any) -> Any: 2587 'Try to narrow the type of the value given.' 2588 2589 if isinstance(x, float): 2590 return int(x) if floor(x) == x else x 2591 2592 if not isinstance(x, str): 2593 return x 2594 2595 try: 2596 return loads(x) 2597 except Exception: 2598 pass 2599 2600 try: 2601 return int(x) 2602 except Exception: 2603 pass 2604 2605 try: 2606 return float(x) 2607 except Exception: 2608 pass 2609 2610 return x 2611 2612 autocast = retype 2613 mold = retype 2614 molded = retype 2615 narrow = retype 2616 narrowed = retype 2617 recast = retype 2618 recasted = retype 2619 remold = retype 2620 remolded = retype 2621 retyped = retype 2622 2623 def revcompose(*what: Callable) -> Callable: 2624 def composite(x: Any) -> Any: 2625 for f in reversed(what): 2626 x = f(x) 2627 return x 2628 return composite 2629 2630 rcompose = revcompose 2631 rcomposed = revcompose 2632 revcomposed = revcompose 2633 2634 def revsort(iterable: Iterable, key: Optional[Callable] = None) -> Iterable: 2635 return sorted(iterable, key=key, reverse=True) 2636 2637 revsorted = revsort 2638 2639 # def revsortkv(src: Dict, key: Callable = None) -> Dict: 2640 # if not key: 2641 # key = lambda kv: (kv[1], kv[0]) 2642 # return sortkv(src, key, reverse=True) 2643 2644 def revsortkv(src: Dict, key: Callable = None) -> Dict: 2645 if key is None: 2646 key = lambda x: x[1] 2647 return sortkv(src, key, reverse=True) 2648 2649 revsortedkv = revsortkv 2650 2651 def rstripdecs(s: str) -> str: 2652 ''' 2653 Ignore trailing zero decimals on number-like strings; even ignore 2654 the decimal dot if trailing. 2655 ''' 2656 2657 try: 2658 f = float(s) 2659 if isnan(f) or isinf(f): 2660 return s 2661 2662 dot = s.find('.') 2663 if dot < 0: 2664 return s 2665 2666 s = s.rstrip('0') 2667 return s[:-1] if s.endswith('.') else s 2668 except Exception: 2669 return s 2670 2671 chopdecs = rstripdecs 2672 2673 def scale(x: float, x0: float, x1: float, y0: float, y1: float) -> float: 2674 'Transform a value from a linear domain into another linear one.' 2675 return (y1 - y0) * (x - x0) / (x1 - x0) + y0 2676 2677 rescale = scale 2678 rescaled = scale 2679 scaled = scale 2680 2681 def shortened(s: str, maxlen: int, trailer: str = '') -> str: 2682 'Limit strings to the symbol-count given, including an optional trailer.' 2683 maxlen = max(maxlen, 0) 2684 return s if len(s) <= maxlen else s[:maxlen - len(trailer)] + trailer 2685 2686 def shuffled(x: Any) -> Any: 2687 'Return a shuffled copy of the list given.' 2688 y = copy(x) 2689 shuffle(y) 2690 return y 2691 2692 def split(src: Union[str, Sequence], n: Union[str, int]) -> Iterable: 2693 'Split/break a string/sequence into several chunks/parts.' 2694 2695 if isinstance(src, str) and isinstance(n, str): 2696 return src.split(n) 2697 if not (isinstance(src, (str, Sequence)) and isinstance(n, int)): 2698 raise Exception('unsupported type-pair of arguments') 2699 2700 if n < 1: 2701 return [] 2702 2703 l = len(src) 2704 if l <= n: 2705 return src.split('') if isinstance(src, str) else src 2706 2707 chunks = [] 2708 csize = int(ceil(l / n)) 2709 while len(src) > 0: 2710 chunks.append(src[:csize]) 2711 src = src[csize:] 2712 return chunks 2713 2714 broken = split 2715 splitted = split 2716 splitten = split 2717 2718 def strdrop(x: str, *what: str) -> str: 2719 'Ignore all occurrences of all substrings given.' 2720 2721 for s in what: 2722 x = x.replace(s, '') 2723 return x 2724 2725 strignore = strdrop 2726 2727 def stringify(x: Any) -> str: 2728 'Fancy alias for func dumps, named after JavaScript\'s func.' 2729 return dumps(x, separators=(', ', ': '), allow_nan=False, indent=None) 2730 2731 jsonate = stringify 2732 jsonify = stringify 2733 tojson = stringify 2734 2735 def strafter(x: str, what: str) -> str: 2736 i = x.find(what) 2737 return '' if i < 0 else x[i+len(what):] 2738 2739 def strafterlast(x: str, what: str) -> str: 2740 i = x.rfind(what) 2741 return '' if i < 0 else x[i+len(what):] 2742 2743 def strbefore(x: str, what: str) -> str: 2744 i = x.find(what) 2745 return x if i < 0 else x[:i] 2746 2747 def strbeforelast(x: str, what: str) -> str: 2748 i = x.rfind(what) 2749 return x if i < 0 else x[:i] 2750 2751 def strsince(x: str, what: str) -> str: 2752 i = x.find(what) 2753 return '' if i < 0 else x[i:] 2754 2755 def strsincelast(x: str, what: str) -> str: 2756 i = x.rfind(what) 2757 return '' if i < 0 else x[i:] 2758 2759 def struntil(x: str, what: str) -> str: 2760 i = x.find(what) 2761 return x if i < 0 else x[:i+len(what)] 2762 2763 def struntillast(x: str, what: str) -> str: 2764 i = x.rfind(what) 2765 return x if i < 0 else x[:i+len(what)] 2766 2767 struntilfinal = struntillast 2768 2769 def since(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2770 'Start strings/sequences with a substring/value\'s appearance.' 2771 return (strsince if isinstance(x, str) else itemssince)(x, what) 2772 2773 def sincelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2774 'Start strings/sequences with a substring/value\'s last appearance.' 2775 return (strsincelast if isinstance(x, str) else itemssincelast)(x, what) 2776 2777 sincefinal = sincelast 2778 2779 def sortk(x: Dict, key: Callable = identity, reverse: bool = False) -> Dict: 2780 keys = sorted(x.keys(), key=key, reverse=reverse) 2781 return {k: x[k] for k in keys} 2782 2783 sortkeys = sortk 2784 sortedkeys = sortk 2785 2786 def sortkv(src: Dict, key: Callable = None, reverse: bool = False) -> Dict: 2787 if key is None: 2788 key = lambda x: x[1] 2789 kv = sorted(src.items(), key=key, reverse=reverse) 2790 return {k: v for (k, v) in kv} 2791 2792 sortedkv = sortkv 2793 2794 def squeeze(s: str) -> str: 2795 ''' 2796 A more aggressive way to rid strings of extra spaces which, unlike string 2797 method strip, also turns inner runs of multiple spaces into single ones. 2798 ''' 2799 s = s.strip() 2800 s = spaces_re.sub(' ', s) 2801 s = paddable_tab_re.sub('\t', s) 2802 return s 2803 2804 squeezed = squeeze 2805 2806 def stround(x: Union[int, float], decimals: int = 6) -> str: 2807 'Format numbers into a string with the given decimal-digit count.' 2808 2809 if decimals >= 0: 2810 return f'{x:.{decimals}f}' 2811 else: 2812 return f'{round(x, decimals):.0f}' 2813 2814 def tally(src: Iterable, by: Callable = identity) -> Dict[Any, int]: 2815 ''' 2816 Count all distinct (transformed) values, the result being a dictionary 2817 whose keys are all the transformed values, and whose items are positive 2818 integers. 2819 ''' 2820 2821 if callable(src): 2822 src, by = by, src 2823 2824 tally: Dict[Any, int] = {} 2825 by = loopify(by) 2826 2827 if isinstance(src, dict): 2828 for k, v in src.items(): 2829 dk = by(k, v) 2830 if dk in tally: 2831 tally[dk] += 1 2832 else: 2833 tally[dk] = 1 2834 else: 2835 for i, v in enumerate(src): 2836 dk = by(i, v) 2837 if dk in tally: 2838 tally[dk] += 1 2839 else: 2840 tally[dk] = 1 2841 return tally 2842 2843 tallied = tally 2844 2845 def transpose(src: Any) -> Any: 2846 'Turn lists/objects inside-out like socks, so to speak.' 2847 2848 if isinstance(src, dict): 2849 return { v: k for k, v in src.items() } 2850 2851 if not arrayish(src): 2852 msg = 'transpose only supports objects or iterables of objects' 2853 raise ValueError(msg) 2854 2855 kv: Dict[Any, Any] = {} 2856 seq: List[Any] = [] 2857 2858 for e in src: 2859 if isinstance(e, dict): 2860 for k, v in e.items(): 2861 if k in kv: 2862 kv[k].append(v) 2863 else: 2864 kv[k] = [v] 2865 elif isinstance(e, Iterable): 2866 for i, v in enumerate(e): 2867 if i < len(seq): 2868 seq[i].append(v) 2869 else: 2870 seq.append([v]) 2871 else: 2872 msg = 'transpose(...): not all items are iterables/objects' 2873 raise ValueError(msg) 2874 2875 if len(kv) > 0 and len(seq) > 0: 2876 msg = 'transpose(...): mix of iterables and objects not supported' 2877 raise ValueError(msg) 2878 return kv if len(seq) == 0 else seq 2879 2880 tr = transpose 2881 transp = transpose 2882 transposed = transpose 2883 2884 def trap(x: Callable, y: Union[Callable[[Exception], Any], Any] = None) -> Any: 2885 'Try running a func, handing exceptions over to a fallback func.' 2886 2887 try: 2888 return x() if callable(x) else x 2889 except Exception as e: 2890 if callable(y): 2891 nargs = required_arg_count(y) 2892 return y(e) if nargs == 1 else y() 2893 else: 2894 return y 2895 2896 catch = trap 2897 catched = trap 2898 caught = trap 2899 noerr = trap 2900 noerror = trap 2901 noerrors = trap 2902 safe = trap 2903 save = trap 2904 saved = trap 2905 trapped = trap 2906 2907 def tsv(x: str, fn: Union[Callable, None] = None) -> Any: 2908 if fn is None: 2909 return x.split('\t') 2910 if callable(x): 2911 x, fn = fn, x 2912 return fn(x.split('\t')) 2913 2914 def typename(x: Any) -> str: 2915 if x is None: 2916 return 'null' 2917 if isinstance(x, bool): 2918 return 'boolean' 2919 if isinstance(x, str): 2920 return 'string' 2921 if isinstance(x, (int, float)): 2922 return 'number' 2923 if isinstance(x, (list, tuple)): 2924 return 'array' 2925 if isinstance(x, dict): 2926 return 'object' 2927 return type(x).__name__ 2928 2929 def typeof(x: Any) -> str: 2930 'Get a value\'s JS-like typeof type-string.' 2931 2932 if callable(x): 2933 return 'function' 2934 2935 return { 2936 bool: 'boolean', 2937 int: 'number', 2938 float: 'number', 2939 str: 'string', 2940 }.get(type(x), 'object') 2941 2942 def unixify(s: str) -> str: 2943 ''' 2944 Make plain-text `unix-style`, ignoring a leading UTF-8 BOM if present, 2945 and turning any/all CRLF byte-pairs into line-feed bytes. 2946 ''' 2947 s = s.lstrip('\xef\xbb\xbf') 2948 return s.replace('\r\n', '\n') if '\r\n' in s else s 2949 2950 def unquoted(s: str) -> str: 2951 'Ignore surrounding quotes in a string.' 2952 2953 if s.startswith('"') and s.endswith('"'): 2954 return s[1:-1] 2955 if s.startswith('\'') and s.endswith('\''): 2956 return s[1:-1] 2957 if s.startswith('`') and s.endswith('`'): 2958 return s[1:-1] 2959 if s.startswith('”') and s.endswith('“'): 2960 return s[1:-1] 2961 if s.startswith('“') and s.endswith('”'): 2962 return s[1:-1] 2963 return s 2964 2965 dequote = unquoted 2966 dequoted = unquoted 2967 2968 def until(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2969 'End strings/sequences with a substring/value\'s appearance.' 2970 return (struntil if isinstance(x, str) else itemsuntil)(x, what) 2971 2972 def untillast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2973 'End strings/sequences with a substring/value\'s last appearance.' 2974 return (struntillast if isinstance(x, str) else itemsuntillast)(x, what) 2975 2976 untilfinal = untillast 2977 2978 2979 def wait(seconds: Union[int, float], result: Any) -> Any: 2980 'Wait the given number of seconds, before returning its latter arg.' 2981 2982 t = (int, float) 2983 if (not isinstance(seconds, t)) and isinstance(result, t): 2984 seconds, result = result, seconds 2985 sleep(seconds) 2986 return result 2987 2988 delay = wait 2989 2990 def wat(*args) -> None: 2991 'What Are These (wat) shows help/doc messages for funcs given to it.' 2992 2993 from pydoc import doc 2994 2995 c = 0 2996 w = stderr 2997 2998 for e in args: 2999 if not callable(e): 3000 continue 3001 3002 if c > 0: 3003 print(file=w) 3004 3005 print(f'\x1b[48;5;253m\x1b[38;5;26m{e.__name__:80}\x1b[0m', file=w) 3006 doc(e, output=w) 3007 c += 1 3008 3009 return Skip() 3010 3011 def wit(*args) -> None: 3012 'What Is This (wit) shows help/doc messages for funcs given to it.' 3013 return wat(*args) 3014 3015 def zoom(x: Any, *keys_indices) -> Any: 3016 for k in keys_indices: 3017 # allow int-indexing dicts the same way lists/tuples can be 3018 if isinstance(x, dict) and isinstance(k, int): 3019 l = len(x) 3020 if i < 0: 3021 i += l 3022 if i < 0 or i >= len(x): 3023 x = None 3024 continue 3025 for i, e in enumerate(x.values()): 3026 if i == k: 3027 x = e 3028 break 3029 continue 3030 3031 # regular key/index access for dicts/lists/tuples 3032 x = x[k] 3033 3034 return x 3035 3036 3037 no_input_opts = ( 3038 '=', '-None', '--None', '-nil', '--nil', '-noinput', '--noinput', 3039 '-no-input', '--no-input', '-none', '--none', '-null', '--null', 3040 '-null-input', '--null-input', '-nullinput', '--nullinput', '--n', 3041 ) 3042 json0_opts = ( 3043 '-c', '--c', '-compact', '--compact', '-j0', '--j0', 3044 '-json0', '--json0', '-json-0', '--json-0', 3045 ) 3046 traceback_opts = ( 3047 '-t', '--t', '-trace', '--trace', '-traceback', '--traceback', 3048 ) 3049 profile_opts = ('-p', '--p', '-prof', '--prof', '-profile', '--profile') 3050 dot_opts = ('-d', '--d', '-dot', '--dot', '-dots', '--dots') 3051 zoom_opts = ('-z', '--z', '-zj', '--zj', '-zoom', '--zoom') 3052 3053 3054 # no args or a leading help-option arg means show the help message and quit 3055 if len(argv) == 3: 3056 if argv[1] in ('-h', '--h', '-help', '--help') and argv[2] in zoom_opts: 3057 print(zj_info_msg.strip(), file=stderr) 3058 exit(0) 3059 3060 3061 args = argv[1:] 3062 load_input = True 3063 dot_input = False 3064 trace_exceptions = False 3065 profile_eval = False 3066 compact_output = False 3067 expression = None 3068 name = '' 3069 3070 # handle all other leading options; the explicit help options are 3071 # handled earlier in the script 3072 while len(args) > 0: 3073 if args[0] in no_input_opts: 3074 load_input = False 3075 args = args[1:] 3076 elif args[0] in json0_opts: 3077 compact_output = True 3078 args = args[1:] 3079 elif args[0] in traceback_opts: 3080 trace_exceptions = True 3081 args = args[1:] 3082 elif args[0] in profile_opts: 3083 profile_eval = True 3084 args = args[1:] 3085 elif args[0] in dot_opts: 3086 dot_input = True 3087 args = args[1:] 3088 elif args[0] in zoom_opts: 3089 # if len(args) < 2: 3090 # print(zj_info_msg.strip(), file=stderr) 3091 # exit(0) 3092 3093 try: 3094 z = load(stdin) 3095 z = zj_zoom(z, tuple(args[1:])) 3096 ind = None if compact_output else 2 3097 seps = (',', ':') if compact_output else (',', ': ') 3098 dump(z, stdout, indent=ind, separators=seps, allow_nan=False) 3099 stdout.write('\n') 3100 except BrokenPipeError: 3101 # quit quietly, instead of showing a confusing error message 3102 stderr.close() 3103 except KeyboardInterrupt: 3104 exit(2) 3105 except Exception as e: 3106 if trace_exceptions: 3107 raise e 3108 s = str(e) 3109 s = s if s else '<generic exception>' 3110 print(f'\x1b[31m{s}\x1b[0m', file=stderr) 3111 exit(1) 3112 3113 exit(0) 3114 else: 3115 break 3116 3117 if len(args) > 2 and load_input: 3118 print('\x1b[31mmultiple inputs not allowed\x1b[0m', file=stderr) 3119 exit(1) 3120 3121 if len(args) == 1: 3122 expression = args[0] 3123 args = args[1:] 3124 elif len(args) > 1: 3125 expression = args[0] 3126 name = args[1] 3127 args = args[1:] 3128 3129 if expression is None: 3130 print(info.strip(), file=stderr) 3131 exit(0) 3132 3133 glo = globals() 3134 for e in (physics, symbols, units): 3135 for _k, _v in e.__dict__.items(): 3136 if not _k in glo: 3137 glo[_k] = _v 3138 3139 try: 3140 # load JSON into variable `v`, unless input was explicitly disabled 3141 v = None 3142 if load_input: 3143 try: 3144 if name == '' or name == '-': 3145 v = load(stdin) 3146 elif seems_url(name): 3147 from urllib.request import urlopen 3148 with urlopen(name) as inp: 3149 v = load(inp) 3150 else: 3151 with open(name, encoding='utf-8') as inp: 3152 v = load(inp) 3153 except Exception as e: 3154 raise Exception(f'JSON-input error: {e}') 3155 3156 if dot_input: 3157 v = dotate(v) 3158 3159 # offer several aliases for main variable `v`; the intuitive 3160 # `in` (short for `input`) is a keyword, so it's not available 3161 data = value = d = dat = val = v 3162 3163 # transform data using the formula/expression given, handling 3164 # single dots as identity operations 3165 exec = disabled_exec 3166 open = disabled_open 3167 if not expression or expression == '.': 3168 expression = 'data' 3169 expression = compile_py(expression, expression, 'eval') 3170 if profile_eval: 3171 from cProfile import Profile 3172 # using a profiler in a `with` context adds many irrelevant 3173 # entries to its output 3174 prof = Profile() 3175 prof.enable() 3176 v = eval(expression) 3177 if result_needs_fixing(v): 3178 v = fix_result(v, data) 3179 prof.disable() 3180 prof.print_stats() 3181 else: 3182 v = eval(expression) 3183 if result_needs_fixing(v): 3184 v = fix_result(v, data) 3185 3186 # emit result as JSON 3187 try: 3188 ind = None if compact_output else 2 3189 seps = (',', ':') if compact_output else (',', ': ') 3190 dump(v, stdout, indent=ind, separators=seps, allow_nan=False) 3191 stdout.write('\n') 3192 except Exception as e: 3193 raise Exception(f'JSON-output error: {e}') 3194 except BrokenPipeError: 3195 # quit quietly, instead of showing a confusing error message 3196 stderr.close() 3197 except KeyboardInterrupt: 3198 exit(2) 3199 except Exception as e: 3200 if trace_exceptions: 3201 raise e 3202 s = str(e) 3203 s = s if s else '<generic exception>' 3204 print(f'\x1b[31m{s}\x1b[0m', file=stderr) 3205 exit(1)