File: tj.py 1 #!/usr/bin/python3 2 3 # The MIT License (MIT) 4 # 5 # Copyright © 2024 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the “Software”), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tj [options...] [python expression] [filepath/URI...] 28 29 30 Transform Json loads JSON data, runs a Python expression on it, and emits 31 the result as JSON. Parsed input-data are available to the expression as 32 any of the variables named `v`, `value`, `d`, and `data`. 33 34 If no file/URI is given, it loads JSON data from its standard input. If the 35 argument before the expression is a single equals sign (a `=`, without the 36 quotes), no data are read/parsed, and the expression is evaluated as given. 37 38 Options, where leading double-dashes are also allowed, except for alias `=`: 39 40 -c compact single-line JSON output (JSON-0) 41 -compact same as -c 42 -j0 same as -c 43 -json0 same as -c 44 -json-0 same as -c 45 46 -h show this help message 47 -help same as -h 48 49 -nil don't read any input 50 -no-input same as -nil 51 -noinput same as -nil 52 -none same as -nil 53 -null same as -nil 54 -null-input same as -nil 55 -nullinput same as -nil 56 = same as -nil 57 58 -d recursively make dictionary values dot-accessible 59 -dot same as -d 60 -dots same as -d 61 62 -p show a performance/time-profile of the expression run 63 -prof same as -p 64 -profile same as -p 65 66 -t show a full traceback of this script for exceptions 67 -trace same as -t 68 -traceback same as -t 69 70 -z zoom JSON value read from stdin, using all the keys given 71 -zj same as -z 72 -zoom same as -z 73 74 75 Extra Functions 76 77 after(x, y) ignore items until the one given; for strings and sequences 78 afterfinal(x, y) backward counterpart of func after 79 afterlast(x, y) same as func afterfinal 80 arrayish(x) check if value is a list, a tuple, or a generator 81 basename(s) get the final/file part of a pathname 82 before(x, y) ignore items since the one given; for strings and sequences 83 beforefinal(x, y) backward counterpart of func before 84 beforelast(x, y) same as func beforefinal 85 chunk(x, size) split/resequence items into chunks of the length given 86 chunked(x, size) same as func chunk 87 compose(*args) make a func which chain-calls all funcs given 88 composed(*args) same as func compose 89 cond(*args) expression-friendly fully-evaluated if-else chain 90 debase64(s) decode base64 strings, including data-URIs 91 dedup(x) ignore later (re)occurrences of values in a sequence 92 dejson(x, f=None) safe parse JSON from strings 93 denan(x, y) turn a floating-point NaN values into the fallback given 94 denil(*args) return the first non-null/none value among those given 95 denone(*args) same as func denil 96 denull(*args) same as func denil 97 dirname(s) get the folder/directory/parent part of a pathname 98 dive(x, f) transform value in depth-first-recursive fashion 99 divebin(x, y, f) binary (2-input) version of recursive-transform func dive 100 drop(x, *what) ignore keys or substrings; for strings, dicts, dict-lists 101 dropped(x, *v) same as func drop 102 each(x, f) generalization of built-in func map 103 endict(x) turn non-dictionary values into dicts with string keys 104 enfloat(x, f=nan) turn values into floats, offering a fallback on failure 105 enint(x, f=None) turn values into ints, offering a fallback on failure 106 enlist(x) turn non-list values into lists 107 entuple(x) turn non-tuple values into tuples 108 ext(s) return the file-extension part of a pathname, if available 109 fields(s) split fields AWK-style from the string given 110 filtered(x, f) same as func keep 111 flat(*args) flatten everything into an unnested sequence 112 fromto(x, y, ?f) sequence integers, end-value included 113 group(x, ?by) group values into dicts of lists; optional transform func 114 grouped(x, ?by) same as func group 115 harden(f, v) make funcs which return values instead of exceptions 116 hardened(f, v) same as func harden 117 countif(x, f) count how many values make the func given true-like 118 idiota(x, ?f) dict-counterpart of func iota 119 ints(x, y, ?f) make sequences of increasing integers, which include the end 120 iota(x, ?f) make an integer sequence from 1 up to the number given 121 join(x, y) join values into a string; make a dict from keys and values 122 json0(x) turn a value into its smallest JSON-string representation 123 json2(x) turn a value into a 2-space-indented multi-line JSON string 124 jsonl(x) turn a value into a sequence of single-line (JSONL) strings 125 keep(x, pred) generalization of built-in func filter 126 kept(x, pred) same as func keep 127 links(x) auto-detect all hyperlink-like (HTTP/HTTPS) substrings 128 mapped(x, f) same as func each 129 number(x) try to parse as an int, on failure try to parse as a float 130 numbers(x) auto-detect all numbers in the value given 131 numstats(x) calculate various `single-pass` numeric stats 132 once(x, y=None) avoid returning the same value more than once; stateful func 133 pick(x, *what) keep only the keys given; works on dicts, or dict-sequences 134 picked(x, *what) same a func pick 135 plain(s) ignore ANSI-style sequences in strings 136 quoted(s, q='"') surround a string with the (optional) quoting-symbol given 137 recover(*args) recover from exceptions with a fallback value 138 reject(x, pred) generalization of built-in func filter, with opposite logic 139 since(x, y) ignore items before the one given; for strings and sequences 140 sincefinal(x, y) backward counterpart of func since 141 sincelast(x, y) same as func sincefinal 142 split(x, y) split string by separator; split sequence into several ones 143 squeeze(s) strip/trim a string, squishing inner runs of spaces 144 stround(x, d=6) format numbers into decimal-number strings 145 tally(x, ?by) count/tally values, using an optional transformation func 146 tallied(x, ?by) same as func tally 147 trap(x, f=None) try running a func, handing exceptions to a fallback func 148 trycall(*args) same as func recover 149 unique(x) same as func dedup 150 uniqued(x) same as func dedup 151 unjson(x, f=None) same as func dejson 152 unquoted(s) ignore surrounding quotes, if present 153 until(x, y) ignore items after the one given; for strings and sequences 154 untilfinal(x, y) backward counterpart of func until 155 untillast(x, y) same as func untilfinal 156 wait(seconds, x) wait the given number of seconds, before returning a value 157 wat(*args) What Are These (wat) shows help/doc messages for funcs 158 159 160 Examples 161 162 # numbers from 0 to 5; no input is read/used 163 tj = 'range(6)' 164 165 # using bases 1 to 5, find all their powers up to the 4th 166 tj = '((n**p for p in range(1, 4+1)) for n in range(1, 6))' 167 168 # keep only the last 2 items from the input 169 tj = 'range(1, 6)' | tj 'data[-2:]' 170 171 # chunk/regroup input items into arrays of up to 3 items each 172 tj = 'range(1, 8)' | tj 'chunk(data, 3)' 173 174 # ignore all items before the first one with just a 5 in it 175 tj = 'range(8)' | tj 'since(data, 5)' 176 177 # ignore errors/exceptions, in favor of a fallback value 178 tj = 'safe(lambda: 2 * float("no way"), "fallback value")' 179 180 # ignore errors/exceptions, calling a fallback func with the exception 181 tj = 'safe(lambda: 2 * float("no way"), lambda err: str(err))' 182 183 # use dot-syntax on JSON data 184 tj = '{"abc": {"xyz": 123}}' | tj -d 'data.abc.xyz' 185 186 # use dot-syntax on JSON data; keywords as properties are syntax-errors 187 tj = '{"abc": {"def": 123}}' | tj -d 'data.abc["def"]' 188 189 # func results are automatically called on the input 190 tj = '{"abc": 123, "def": 456}' | tj len 191 ''' 192 193 194 from sys import argv, exit, stderr, stdin, stdout 195 196 197 if __name__ != '__main__': 198 print('don\'t import this script, run it directly instead', file=stderr) 199 exit(1) 200 201 # no args or a leading help-option arg means show the help message and quit 202 help_opts = ('-h', '--h', '-help', '--help') 203 if len(argv) < 2 or (len(argv) == 2 and argv[1] in help_opts): 204 print(info.strip(), file=stderr) 205 exit(0) 206 207 208 from io import StringIO 209 from itertools import islice 210 from json import load 211 212 from typing import \ 213 AbstractSet, Annotated, Any, AnyStr, \ 214 AsyncContextManager, AsyncGenerator, AsyncIterable, AsyncIterator, \ 215 Awaitable, BinaryIO, ByteString, Callable, cast, \ 216 ClassVar, Collection, Container, \ 217 ContextManager, Coroutine, Deque, Dict, Final, \ 218 final, ForwardRef, FrozenSet, Generator, Generic, get_args, get_origin, \ 219 get_type_hints, Hashable, IO, ItemsView, \ 220 Iterable, Iterator, KeysView, List, Literal, Mapping, \ 221 MappingView, Match, MutableMapping, MutableSequence, MutableSet, \ 222 NamedTuple, NewType, no_type_check, no_type_check_decorator, \ 223 NoReturn, Optional, overload, \ 224 Protocol, Reversible, \ 225 runtime_checkable, Sequence, Set, Sized, SupportsAbs, \ 226 SupportsBytes, SupportsComplex, SupportsFloat, SupportsIndex, \ 227 SupportsInt, SupportsRound, Text, TextIO, Tuple, Type, \ 228 TypedDict, TypeVar, \ 229 TYPE_CHECKING, Union, ValuesView 230 try: 231 from typing import \ 232 assert_never, assert_type, clear_overloads, Concatenate, \ 233 dataclass_transform, get_overloads, is_typeddict, LiteralString, \ 234 Never, NotRequired, ParamSpec, ParamSpecArgs, ParamSpecKwargs, \ 235 Required, reveal_type, Self, TypeAlias, TypeGuard, TypeVarTuple, \ 236 Unpack 237 from typing import \ 238 AwaitableGenerator, override, TypeAliasType, type_check_only 239 except Exception: 240 pass 241 242 243 def conforms(x: Any) -> bool: 244 ''' 245 Check if a value is JSON-compatible, which includes checking values 246 recursively, in case of composite/nestable values. 247 ''' 248 249 if x is None or isinstance(x, (bool, int, str)): 250 return True 251 if isinstance(x, float): 252 return not (isnan(x) or isinf(x)) 253 if isinstance(x, (list, tuple)): 254 return all(conforms(e) for e in x) 255 if isinstance(x, dict): 256 return all(conforms(k) and conforms(v) for k, v in x.items()) 257 return False 258 259 260 def seems_url(s: str) -> bool: 261 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 262 return any(s.startswith(p) for p in protocols) 263 264 265 def result_needs_fixing(x: Any) -> bool: 266 ''' 267 See if func fix_result needs to be called: avoiding that can speed 268 things up and save memory, when a composite value is big enough. 269 ''' 270 271 if x is None or isinstance(x, (bool, int, float, str)): 272 return False 273 rec = result_needs_fixing 274 if isinstance(x, dict): 275 return any(rec(k) or rec(v) for k, v in x.items()) 276 if isinstance(x, (list, tuple)): 277 return any(rec(e) for e in x) 278 return True 279 280 281 def fix_result(x: Any, default: Any) -> Any: 282 'Adapt a value so it can be output.' 283 284 if x is type: 285 return type(default).__name__ 286 287 # if expression results in a func, auto-call it with the original data 288 if callable(x): 289 c = required_arg_count(x) 290 if c == 1: 291 x = x(default) 292 else: 293 m = f'func auto-call only works with 1-arg funcs (func wanted {c})' 294 raise Exception(m) 295 296 if x is None or isinstance(x, (bool, int, float, str)): 297 return x 298 299 rec = fix_result 300 301 if isinstance(x, dict): 302 return { 303 rec(k, default): rec(v, default) for k, v in x.items() if not 304 (isinstance(k, Skip) or isinstance(v, Skip)) 305 } 306 if isinstance(x, Iterable): 307 return tuple(rec(e, default) for e in x if not isinstance(e, Skip)) 308 309 if isinstance(x, Dottable): 310 return rec(x.__dict__, default) 311 if isinstance(x, DotCallable): 312 return rec(x.value, default) 313 314 if isinstance(x, Exception): 315 raise x 316 317 return None if isinstance(x, Skip) else str(x) 318 319 320 def disabled_exec(*args, **kwargs) -> None: 321 _ = args 322 _ = kwargs 323 raise Exception('built-in func `exec` is disabled') 324 325 326 def disabled_open(*args, **kwargs) -> None: 327 _ = args 328 _ = kwargs 329 raise Exception('built-in func `open` is disabled') 330 331 332 from re import compile as zj_compile_re 333 334 335 zj_info_msg = ''' 336 zj [keys/indices...] 337 338 339 Zoom Json digs into a subset of valid JSON input, using the given mix of 340 keys and array-indices, the latter being either 0-based or negative, to 341 index backward from the ends of arrays. 342 343 Zooming on object keys is first tried as an exact key-match, failing that 344 as a case-insensitive key-match (first such match): when both approaches 345 fail, if the key is a valid integer, the key at the (even negative) index 346 given is used. 347 348 Invalid array-indices and missing object-keys result in null values, when 349 none of the special keys/fallbacks shown later apply. 350 351 You can slice arrays the exclusive/go/python way using index-pairs with a 352 `:` between the start/end pair, as long as it's a single argument; you can 353 even use `..` as the index-pair separator to include the stop index in the 354 result. Either way, as with go/python, you can omit either of the indices 355 when slicing. 356 357 Special key `.` acts as implicit loops on arrays, and even objects without 358 that specific key: in the unlikely case that an object has `.` as one of 359 its keys, you can use one of loop-fallback aliases, shown later. 360 361 Another special key is `+` (no quotes): when used, the rest of the keys 362 are used `in parallel`, allowing multiple picks from the current value. 363 When picking array items, you can also use either type (`:` or `..`) of 364 slicing, even mixing it with individual indices. 365 366 Similar to `+`, the `-` fallback-key drops keys, which means all items are 367 picked, except for those mentioned after the `-`. 368 369 Unlike the looping special key, after the first `+` special-key, all keys 370 following it, special or not, are picked normally. 371 372 In case any of the special keys are actual keys in the data loaded, some 373 aliases are available: 374 375 . /. ./ :. .: 376 + /+ +/ :+ +: 377 - /- -/ :- -: 378 379 .i :i .info :info :info: 380 .k :k .keys :keys :keys: 381 .t :t .type :type :type: 382 .l :l .len .length :len :len: :length :length: 383 384 These aliases allow using the special functionality even on objects whose 385 keys match some of these special names, as it's extremely unlikely data use 386 all aliases as actual keys at any level. 387 388 The only input supported is valid JSON coming from standard-input: there's 389 no way to load files using their names. To load data from files/URIs use 390 tools like `cat` or `curl`, and pipe their output into this tool. 391 ''' 392 393 zj_slice_re = zj_compile_re('''^(([+-]?[0-9]+)?)(:|\.\.)(([+-]?[0-9]+)?)$''') 394 395 396 def zj_zoom(data: Any, keys: Tuple[str, ...]) -> Any: 397 eval_due = False 398 pyf = ('.pyl', ':pyl', 'pyl:', ':pyl:', '.pyf', ':pyf', 'pyf:', ':pyf:') 399 400 for i, k in enumerate(keys): 401 try: 402 if eval_due: 403 data = eval(k)(data) 404 eval_due = False 405 continue 406 407 if isinstance(data, dict): 408 m = zj_match_key(data, k) 409 if m in data: 410 data = data[m] 411 continue 412 m = zj_slice_re.match(k) 413 if m: 414 data = {k: data[k] for k in zj_match_keys(data, k)} 415 continue 416 417 if isinstance(data, (list, tuple)): 418 try: 419 i = int(k) 420 l = len(data) 421 data = data[i] if -l <= i < l else None 422 continue 423 except Exception: 424 m = zj_slice_re.match(k) 425 if m: 426 data = [data[i] for i in zj_match_indices(data, k)] 427 continue 428 429 if k in pyf: 430 eval_due = True 431 continue 432 433 if k in ('.', '/.', './', ':.', '.:'): 434 if isinstance(data, dict): 435 rest = tuple(keys[i + 1:]) 436 return {k: zj_zoom(v, rest) for k, v in data.items()} 437 if isinstance(data, (list, tuple)): 438 rest = tuple(keys[i + 1:]) 439 return tuple(zj_zoom(v, rest) for v in data) 440 441 # doing nothing amounts to an identity-op for simple values 442 continue 443 444 fn = zj_final_fallbacks.get(k, None) 445 if fn: 446 return fn(data, tuple(keys[i + 1:])) 447 448 fn = zj_fallbacks.get(k, None) 449 if fn: 450 data = fn(data) 451 continue 452 453 if isinstance(data, (dict, list, tuple)): 454 data = None 455 continue 456 457 kind = zj_type(data) 458 msg = f'value of type {kind} has no properties to zoom into' 459 raise Exception(msg) 460 except Exception as e: 461 key_path = ' > '.join(islice(keys, None, i + 1)) 462 raise Exception(f'{key_path}: {e}') 463 464 return data 465 466 467 def zj_match_key(src: Dict, key: str) -> str: 468 if key in src: 469 return key 470 471 low = key.casefold() 472 for k in src.keys(): 473 if low == k.casefold(): 474 return k 475 476 try: 477 i = int(key) 478 l = len(src) 479 if i < 0: 480 i += l 481 if i < 0 or i >= l: 482 return None 483 for j, k in enumerate(src.keys()): 484 if i == j: 485 return k 486 except Exception: 487 return key 488 return key 489 490 491 def zj_match_keys(src: Any, key: str) -> Iterable: 492 if isinstance(src, (list, tuple)): 493 yield from zj_match_indices(src, key) 494 yield from zj_match_fallbacks(src, key) 495 return 496 497 if isinstance(src, dict): 498 if key in src: 499 yield key 500 return 501 502 low = key.casefold() 503 for k in src.keys(): 504 if low == k.casefold(): 505 yield k 506 return 507 508 yield from zj_match_indices(src, key) 509 yield from zj_match_fallbacks(src, key) 510 return 511 512 yield from zj_match_fallbacks(src, key) 513 514 515 def zj_match_indices(src: Any, key: str) -> Iterable: 516 try: 517 i = int(key) 518 519 if isinstance(src, (list, tuple)): 520 l = len(src) 521 yield src[i] if -l <= i < l else None 522 return 523 524 if isinstance(src, dict): 525 l = len(src) 526 if i < 0: 527 i += l 528 if i < 0 or i >= l: 529 return 530 531 for j, k in enumerate(src.keys()): 532 if i == j: 533 yield k 534 return 535 536 return 537 except Exception: 538 pass 539 540 m = zj_slice_re.match(key) 541 if not m: 542 return 543 544 l = len(src) 545 546 (start, _, kind, stop, _) = m.groups() 547 start = int(start) if start != '' else 0 548 stop = int(stop) if stop != '' else l 549 550 if start < 0: 551 start += l 552 start = max(start, 0) 553 if stop < 0: 554 stop += l 555 stop = min(stop, l) 556 if kind == '..': 557 stop += 1 558 stop = min(stop, l) 559 560 if start > stop: 561 return 562 if (start < 0 and stop < 0) or (start >= l and stop >= l): 563 return 564 565 566 if isinstance(src, dict): 567 for i, k in enumerate(src.keys()): 568 if i >= stop: 569 return 570 if start <= i: 571 yield k 572 return 573 574 if isinstance(src, (list, tuple)): 575 yield from range(start, stop) 576 return 577 578 579 580 def zj_match_fallbacks(src: Any, key: str) -> Iterable: 581 fn = zj_fallbacks.get(key, None) 582 if fn: 583 yield fn(src) 584 585 586 def zj_help(*_) -> NoReturn: 587 print(zj_info_msg.strip(), file=stderr) 588 exit(1) 589 590 591 def zj_keys(src: Any) -> Any: 592 if isinstance(src, dict): 593 return tuple(src.keys()) 594 if isinstance(src, (list, tuple)): 595 return tuple(range(len(src))) 596 return None 597 598 599 def zj_info(x: Any) -> str: 600 if isinstance(x, dict): 601 return f'object ({len(x)} items)' 602 if isinstance(x, (list, tuple)): 603 return f'array ({len(x)} items)' 604 return zj_type(x) 605 606 607 def zj_type(x: Any) -> str: 608 return { 609 type(None): 'null', 610 bool: 'boolean', 611 dict: 'object', 612 float: 'number', 613 int: 'number', 614 str: 'string', 615 list: 'array', 616 tuple: 'array', 617 }.get(type(x), 'other') 618 619 620 zj_fallbacks: Dict[str, Callable] = { 621 '.h': zj_help, 622 '.help': zj_help, 623 ':h': zj_help, 624 ':help': zj_help, 625 ':help:': zj_help, 626 '.i': zj_info, 627 '.info': zj_info, 628 ':i': zj_info, 629 ':info': zj_info, 630 ':info:': zj_info, 631 '.k': zj_keys, 632 '.keys': zj_keys, 633 ':keys': zj_keys, 634 ':keys:': zj_keys, 635 '.l': len, 636 '.len': len, 637 '.length': len, 638 ':l': len, 639 ':len': len, 640 ':length': len, 641 ':len:': len, 642 ':length:': len, 643 '.t': zj_type, 644 '.type': zj_type, 645 ':t': zj_type, 646 ':type': zj_type, 647 ':type:': zj_type, 648 } 649 650 651 def zj_pick(src: Any, keys: Tuple[str, ...]) -> Any: 652 if isinstance(src, dict): 653 picked = {} 654 for k in keys: 655 for k in zj_match_keys(src, k): 656 picked[k] = src[k] 657 return picked 658 659 if isinstance(src, (list, tuple)): 660 picked = [] 661 for k in keys: 662 for i in zj_match_indices(src, k): 663 picked.append(src[i]) 664 return tuple(picked) 665 666 msg = f'can\'t pick properties from value of type {zj_type(src)}' 667 raise Exception(msg) 668 669 670 def zj_drop(src: Any, keys: Tuple[str, ...]) -> Any: 671 if isinstance(src, dict): 672 avoid = set() 673 for k in keys: 674 for k in zj_match_keys(src, k): 675 avoid.add(k) 676 return {k: v for k, v in src.items() if not k in avoid} 677 678 if isinstance(src, (list, tuple)): 679 l = len(src) 680 avoid = set() 681 for k in keys: 682 for i in zj_match_indices(src, k): 683 avoid.add(i if i >= 0 else i + l) 684 return tuple(v for i, v in enumerate(src) if not i in avoid) 685 686 msg = f'can\'t drop properties from value of type {zj_type(src)}' 687 raise Exception(msg) 688 689 690 zj_final_fallbacks: Dict[str, Callable] = { 691 '+': zj_pick, 692 ':+:': zj_pick, 693 ':+': zj_pick, 694 '+:': zj_pick, 695 '/+': zj_pick, 696 '+/': zj_pick, 697 698 '-': zj_drop, 699 ':-:': zj_drop, 700 ':-': zj_drop, 701 '-:': zj_drop, 702 '/-': zj_drop, 703 '-/': zj_drop, 704 } 705 706 707 from base64 import \ 708 standard_b64encode, standard_b64decode, \ 709 standard_b64encode as base64bytes, standard_b64decode as debase64bytes 710 711 from collections import \ 712 ChainMap, Counter, defaultdict, deque, namedtuple, OrderedDict, \ 713 UserDict, UserList, UserString 714 715 from copy import copy, deepcopy 716 717 from datetime import \ 718 MAXYEAR, MINYEAR, date, datetime, time, timedelta, timezone, tzinfo 719 try: 720 from datetime import now, UTC 721 except Exception: 722 now = lambda: datetime(2000, 1, 1).now() 723 724 from decimal import Decimal, getcontext 725 726 from difflib import \ 727 context_diff, diff_bytes, Differ, get_close_matches, HtmlDiff, \ 728 IS_CHARACTER_JUNK, IS_LINE_JUNK, ndiff, restore, SequenceMatcher, \ 729 unified_diff 730 731 from fractions import Fraction 732 733 import functools 734 from functools import \ 735 cache, cached_property, cmp_to_key, get_cache_token, lru_cache, \ 736 namedtuple, partial, partialmethod, recursive_repr, reduce, \ 737 singledispatch, singledispatchmethod, total_ordering, update_wrapper, \ 738 wraps 739 740 from glob import glob, iglob 741 742 try: 743 from graphlib import CycleError, TopologicalSorter 744 except Exception: 745 pass 746 747 from hashlib import \ 748 file_digest, md5, pbkdf2_hmac, scrypt, sha1, sha224, sha256, sha384, \ 749 sha512 750 751 from inspect import getfullargspec, getsource 752 753 import itertools 754 from itertools import \ 755 accumulate, chain, combinations, combinations_with_replacement, \ 756 compress, count, cycle, dropwhile, filterfalse, groupby, islice, \ 757 permutations, product, repeat, starmap, takewhile, tee, zip_longest 758 try: 759 from itertools import pairwise 760 from itertools import batched 761 except Exception: 762 pass 763 764 from json import dump, dumps, loads 765 766 import math 767 Math = math 768 from math import \ 769 acos, acosh, asin, asinh, atan, atan2, atanh, ceil, comb, \ 770 copysign, cos, cosh, degrees, dist, e, erf, erfc, exp, expm1, \ 771 fabs, factorial, floor, fmod, frexp, fsum, gamma, gcd, hypot, inf, \ 772 isclose, isfinite, isinf, isnan, isqrt, lcm, ldexp, lgamma, log, \ 773 log10, log1p, log2, modf, nan, nextafter, perm, pi, pow, prod, \ 774 radians, remainder, sin, sinh, sqrt, tan, tanh, tau, trunc, ulp 775 try: 776 from math import cbrt, exp2 777 except Exception: 778 pass 779 780 power = pow 781 782 import operator 783 784 from pathlib import Path 785 786 from pprint import \ 787 isreadable, isrecursive, pformat, pp, pprint, PrettyPrinter, saferepr 788 789 from random import \ 790 betavariate, choice, choices, expovariate, gammavariate, gauss, \ 791 getrandbits, getstate, lognormvariate, normalvariate, paretovariate, \ 792 randbytes, randint, random, randrange, sample, seed, setstate, \ 793 shuffle, triangular, uniform, vonmisesvariate, weibullvariate 794 795 compile_py = compile # keep built-in func compile for later 796 from re import compile as compile_uncached, Pattern, IGNORECASE 797 798 import statistics 799 from statistics import \ 800 bisect_left, bisect_right, fmean, \ 801 geometric_mean, harmonic_mean, mean, median, \ 802 median_grouped, median_high, median_low, mode, multimode, pstdev, \ 803 pvariance, quantiles, stdev, variance 804 try: 805 from statistics import \ 806 correlation, covariance, linear_regression, mul 807 except Exception: 808 pass 809 810 import string 811 from string import \ 812 Formatter, Template, ascii_letters, ascii_lowercase, ascii_uppercase, \ 813 capwords, digits, hexdigits, octdigits, printable, punctuation, \ 814 whitespace 815 816 alphabet = ascii_letters 817 letters = ascii_letters 818 lowercase = ascii_lowercase 819 uppercase = ascii_uppercase 820 821 from textwrap import dedent, fill, indent, shorten, wrap 822 823 from time import \ 824 altzone, asctime, \ 825 ctime, daylight, get_clock_info, \ 826 gmtime, localtime, mktime, monotonic, monotonic_ns, perf_counter, \ 827 perf_counter_ns, process_time, process_time_ns, \ 828 sleep, strftime, strptime, struct_time, thread_time, thread_time_ns, \ 829 time, time_ns, timezone, tzname 830 try: 831 from time import \ 832 clock_getres, clock_gettime, clock_gettime_ns, clock_settime, \ 833 clock_settime_ns, pthread_getcpuclockid, tzset 834 except Exception: 835 pass 836 837 from unicodedata import \ 838 bidirectional, category, combining, decimal, decomposition, digit, \ 839 east_asian_width, is_normalized, lookup, mirrored, name, normalize, \ 840 numeric 841 842 from urllib.parse import \ 843 parse_qs, parse_qsl, quote, quote_from_bytes, quote_plus, unquote, \ 844 unquote_plus, unquote_to_bytes, unwrap, urldefrag, urlencode, urljoin, \ 845 urlparse, urlsplit, urlunparse, urlunsplit 846 847 848 class Skip: 849 'Custom type which some funcs type-check to skip values in containers.' 850 851 def __init__(self, *args) -> None: 852 pass 853 854 855 # skip is a ready-to-use value which some funcs filter against: this way 856 # filtering values becomes a special case of transforming values 857 skip = Skip() 858 859 # re_cache is used by custom func compile to cache previously-compiled 860 # regular-expressions, which makes them quicker to (re)use in formulas 861 re_cache: Dict[str, Pattern] = {} 862 863 # ire_cache is like re_cache, except it's for case-insensitive regexes 864 ire_cache: Dict[str, Pattern] = {} 865 866 # ansi_style_re detects the most commonly-used ANSI-style sequences, and 867 # is used in func plain 868 ansi_style_re = compile_uncached('''\x1b\[([0-9;]+m|[0-9]*[A-HJKST])''') 869 870 # number_re detects numbers, and is used in func numbers 871 number_re = compile_uncached('''\W(-?[0-9]+(\.[0-9]*)?)\W''') 872 873 # link_re detects web links, and is used in func links 874 link_re_src = 'https?://[A-Za-z0-9+_.:%-]+(/[A-Za-z0-9+_.%/,#?&=-]*)*' 875 link_re = compile_uncached(link_re_src) 876 877 # paddable_tab_re detects single tabs and possible runs of spaces around 878 # them, and is used in func squeeze 879 paddable_tab_re = compile_uncached(' *\t *') 880 881 # seen remembers values already given to func `once` 882 seen = set() 883 884 # commented_re detects strings/lines which start as unix-style comments 885 commented_re = compile_uncached('^ *#') 886 887 # emptyish_re detects empty/emptyish strings/lines, the latter being strings 888 # with only spaces in them 889 emptyish_re = compile_uncached('^ *\r?\n?$') 890 891 # spaces_re detects runs of 2 or more spaces, and is used in func squeeze 892 spaces_re = compile_uncached(' +') 893 894 # awk_sep_re splits like AWK does by default, and is used in func fields 895 awk_sep_re = compile_uncached(' *\t *| +') 896 897 898 # some convenience aliases to commonly-used values 899 900 false = False 901 true = True 902 nil = None 903 nihil = None 904 none = None 905 null = None 906 s = '' 907 908 months = [ 909 'January', 'February', 'March', 'April', 'May', 'June', 910 'July', 'August', 'September', 'October', 'November', 'December', 911 ] 912 913 monweek = [ 914 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 915 'Saturday', 'Sunday', 916 ] 917 918 sunweek = [ 919 'Sunday', 920 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 921 ] 922 923 phy = { 924 'kilo': 1_000, 925 'mega': 1_000_000, 926 'giga': 1_000_000_000, 927 'tera': 1_000_000_000_000, 928 'peta': 1_000_000_000_000_000, 929 'exa': 1_000_000_000_000_000_000, 930 'zetta': 1_000_000_000_000_000_000_000, 931 932 'c': 299_792_458, 933 'kcd': 683, 934 'na': 602214076000000000000000, 935 936 'femto': 1e-15, 937 'pico': 1e-12, 938 'nano': 1e-9, 939 'micro': 1e-6, 940 'milli': 1e-3, 941 942 'e': 1.602176634e-19, 943 'f': 96_485.33212, 944 'h': 6.62607015e-34, 945 'k': 1.380649e-23, 946 'mu': 1.66053906892e-27, 947 948 'ge': 9.7803267715, 949 'gn': 9.80665, 950 } 951 952 physics = phy 953 954 # using literal strings on the cmd-line is often tricky/annoying: some of 955 # these aliases can help get around multiple levels of string-quoting; no 956 # quotes are needed as the script will later make these values accessible 957 # via the property/dot syntax 958 sym = { 959 'amp': '&', 960 'ampersand': '&', 961 'ansiclear': '\x1b[0m', 962 'ansinormal': '\x1b[0m', 963 'ansireset': '\x1b[0m', 964 'apo': '\'', 965 'apos': '\'', 966 'ast': '*', 967 'asterisk': '*', 968 'at': '@', 969 'backquote': '`', 970 'backslash': '\\', 971 'backtick': '`', 972 'ball': '●', 973 'bang': '!', 974 'bigsigma': 'Σ', 975 'block': '█', 976 'bquo': '`', 977 'bquote': '`', 978 'bslash': '\\', 979 'btick': '`', 980 'bullet': '•', 981 'caret': '^', 982 'cdot': '·', 983 'circle': '●', 984 'colon': ':', 985 'comma': ',', 986 'cr': '\r', 987 'crlf': '\r\n', 988 'cross': '×', 989 'cs': ', ', 990 'dash': '—', 991 'dollar': '$', 992 'dot': '.', 993 'dquo': '"', 994 'dquote': '"', 995 'emark': '!', 996 'emdash': '—', 997 'empty': '', 998 'endash': '–', 999 'eq': '=', 1000 'et': '&', 1001 'euro': '€', 1002 'ge': '≥', 1003 'geq': '≥', 1004 'gt': '>', 1005 'hellip': '…', 1006 'hole': '○', 1007 'hyphen': '-', 1008 'infinity': '∞', 1009 'lcurly': '{', 1010 'ldquo': '“', 1011 'ldquote': '“', 1012 'le': '≤', 1013 'leq': '≤', 1014 'lf': '\n', 1015 'lt': '<', 1016 'mdash': '—', 1017 'mdot': '·', 1018 'miniball': '•', 1019 'minus': '-', 1020 'ndash': '–', 1021 'neq': '≠', 1022 'perc': '%', 1023 'percent': '%', 1024 'period': '.', 1025 'plus': '+', 1026 'qmark': '?', 1027 'que': '?', 1028 'rcurly': '}', 1029 'rdquo': '”', 1030 'rdquote': '”', 1031 'sball': '•', 1032 'semi': ';', 1033 'semicolon': ';', 1034 'sharp': '#', 1035 'slash': '/', 1036 'space': ' ', 1037 'square': '■', 1038 'squo': '\'', 1039 'squote': '\'', 1040 'tab': '\t', 1041 'tilde': '~', 1042 'underscore': '_', 1043 'uscore': '_', 1044 'utf8bom': '\xef\xbb\xbf', 1045 'utf16be': '\xfe\xff', 1046 'utf16le': '\xff\xfe', 1047 } 1048 1049 symbols = sym 1050 1051 units = { 1052 'cup2l': 0.23658824, 1053 'floz2l': 0.0295735295625, 1054 'floz2ml': 29.5735295625, 1055 'ft2m': 0.3048, 1056 'gal2l': 3.785411784, 1057 'in2cm': 2.54, 1058 'lb2kg': 0.45359237, 1059 'mi2km': 1.609344, 1060 'mpg2kpl': 0.425143707, 1061 'nmi2km': 1.852, 1062 'oz2g': 28.34952312, 1063 'psi2pa': 6894.757293168, 1064 'ton2kg': 907.18474, 1065 'yd2m': 0.9144, 1066 1067 'mol': 602214076000000000000000, 1068 'mole': 602214076000000000000000, 1069 1070 'hour': 3_600, 1071 'day': 86_400, 1072 'week': 604_800, 1073 1074 'hr': 3_600, 1075 'wk': 604_800, 1076 1077 'kb': 1024, 1078 'mb': 1024**2, 1079 'gb': 1024**3, 1080 'tb': 1024**4, 1081 'pb': 1024**5, 1082 } 1083 1084 # some convenience aliases to various funcs from the python stdlib 1085 geomean = geometric_mean 1086 harmean = harmonic_mean 1087 sd = stdev 1088 popsd = pstdev 1089 var = variance 1090 popvar = pvariance 1091 randbeta = betavariate 1092 randexp = expovariate 1093 randgamma = gammavariate 1094 randlognorm = lognormvariate 1095 randnorm = normalvariate 1096 randweibull = weibullvariate 1097 1098 capitalize = str.capitalize 1099 casefold = str.casefold 1100 center = str.center 1101 # count = str.count 1102 decode = bytes.decode 1103 encode = str.encode 1104 endswith = str.endswith 1105 expandtabs = str.expandtabs 1106 find = str.find 1107 format = str.format 1108 index = str.index 1109 isalnum = str.isalnum 1110 isalpha = str.isalpha 1111 isascii = str.isascii 1112 isdecimal = str.isdecimal 1113 isdigit = str.isdigit 1114 isidentifier = str.isidentifier 1115 islower = str.islower 1116 isnumeric = str.isnumeric 1117 isprintable = str.isprintable 1118 isspace = str.isspace 1119 istitle = str.istitle 1120 isupper = str.isupper 1121 # join = str.join 1122 ljust = str.ljust 1123 lower = str.lower 1124 lowered = str.lower 1125 lstrip = str.lstrip 1126 maketrans = str.maketrans 1127 partition = str.partition 1128 removeprefix = str.removeprefix 1129 removesuffix = str.removesuffix 1130 replace = str.replace 1131 rfind = str.rfind 1132 rindex = str.rindex 1133 rjust = str.rjust 1134 rpartition = str.rpartition 1135 rsplit = str.rsplit 1136 rstrip = str.rstrip 1137 # split = str.split 1138 splitlines = str.splitlines 1139 startswith = str.startswith 1140 strip = str.strip 1141 swapcase = str.swapcase 1142 title = str.title 1143 translate = str.translate 1144 upper = str.upper 1145 uppered = str.upper 1146 zfill = str.zfill 1147 1148 every = all 1149 rev = reversed 1150 reverse = reversed 1151 some = any 1152 1153 length = len 1154 1155 blowtabs = str.expandtabs 1156 hasprefix = str.startswith 1157 hassuffix = str.endswith 1158 ltrim = str.lstrip 1159 stripstart = str.lstrip 1160 trimspace = str.strip 1161 trimstart = str.lstrip 1162 rtrim = str.rstrip 1163 stripend = str.rstrip 1164 trimend = str.rstrip 1165 stripped = str.strip 1166 trim = str.strip 1167 trimmed = str.strip 1168 trimprefix = str.removeprefix 1169 trimsuffix = str.removesuffix 1170 1171 1172 def required_arg_count(f: Callable) -> int: 1173 if isinstance(f, type): 1174 return 1 1175 1176 meta = getfullargspec(f) 1177 n = len(meta.args) 1178 if meta.defaults: 1179 n -= len(meta.defaults) 1180 return n 1181 1182 1183 def identity(x: Any) -> Any: 1184 ''' 1185 Return the value given: this is the default transformer for several 1186 higher-order funcs, which effectively keeps original items as given. 1187 ''' 1188 return x 1189 1190 idem = identity 1191 iden = identity 1192 1193 1194 def after(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1195 'Skip parts of strings/sequences up to the substring/value given.' 1196 return (strafter if isinstance(x, str) else itemsafter)(x, what) 1197 1198 def afterlast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1199 'Skip parts of strings/sequences up to the last substring/value given.' 1200 return (strafterlast if isinstance(x, str) else itemsafterlast)(x, what) 1201 1202 afterfinal = afterlast 1203 1204 def arrayish(x: Any) -> bool: 1205 'Check if a value is array-like enough.' 1206 return isinstance(x, (list, tuple, range, Generator)) 1207 1208 isarrayish = arrayish 1209 1210 def base64(x): 1211 return base64bytes(str(x).encode()).decode() 1212 1213 def basename(s: str) -> str: 1214 'Get a filepath\'s last part, if present.' 1215 return Path(s).name 1216 1217 def before(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1218 'End strings/sequences right before a substring/value\'s appearance.' 1219 return (strbefore if isinstance(x, str) else itemsbefore)(x, what) 1220 1221 def beforelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 1222 'End strings/sequences right before a substring/value\'s last appearance.' 1223 return (strbeforelast if isinstance(x, str) else itemsbeforelast)(x, what) 1224 1225 beforefinal = beforelast 1226 1227 def cases(x: Any, *args: Any) -> Any: 1228 ''' 1229 Simulate a switch statement on a value, using matches/result pairs from 1230 the arguments given; when given an even number of extra args, None is 1231 used as a final fallback result; when given an odd number of extra args, 1232 the last argument is used as a final `default` value, if needed. 1233 ''' 1234 1235 for i in range(0, len(args) - len(args) % 2, 2): 1236 test, res = args[i], args[i+1] 1237 if isinstance(test, (list, tuple)) and x in test: 1238 return res 1239 if isinstance(test, float) and isnan(test) and isnan(x): 1240 return res 1241 if x == test: 1242 return res 1243 return None if len(args) % 2 == 0 else args[-1] 1244 1245 switch = cases 1246 1247 def chunk(items: Iterable, chunk_size: int) -> Iterable: 1248 'Break iterable into chunks, each with up to the item-count given.' 1249 1250 if isinstance(items, str): 1251 n = len(items) 1252 while n >= chunk_size: 1253 yield items[:chunk_size] 1254 items = items[chunk_size:] 1255 n -= chunk_size 1256 if n > 0: 1257 yield items 1258 return 1259 1260 if not isinstance(chunk_size, int): 1261 raise Exception('non-integer chunk-size') 1262 if chunk_size < 1: 1263 raise Exception('non-positive chunk-size') 1264 1265 it = iter(items) 1266 while True: 1267 head = tuple(islice(it, chunk_size)) 1268 if not head: 1269 return 1270 yield head 1271 1272 chunked = chunk 1273 1274 def commented(s: str) -> bool: 1275 'Check if a string starts as a unix-style comment.' 1276 return commented_re.match(s) != None 1277 1278 iscommented = commented 1279 1280 def compile(s: str, case_sensitive: bool = True) -> Pattern: 1281 'Cached regex `compiler`, so it\'s quicker to (re)use in formulas.' 1282 1283 cache = re_cache if case_sensitive else ire_cache 1284 options = 0 if case_sensitive else IGNORECASE 1285 1286 if s in cache: 1287 return cache[s] 1288 e = compile_uncached(s, options) 1289 cache[s] = e 1290 return e 1291 1292 def compose(*what: Callable) -> Callable: 1293 def composite(x: Any) -> Any: 1294 for f in what: 1295 x = f(x) 1296 return x 1297 return composite 1298 1299 composed = compose 1300 lcompose = compose 1301 lcomposed = compose 1302 1303 def cond(*args: Any) -> Any: 1304 ''' 1305 Simulate a chain of if-else statements, using condition/result pairs 1306 from the arguments given; when given an even number of args, None is 1307 used as a final fallback result; when given an odd number of args, the 1308 last argument is used as a final `else` value, if needed. 1309 ''' 1310 1311 for i in range(0, len(args) - len(args) % 2, 2): 1312 if args[i]: 1313 return args[i+1] 1314 return None if len(args) % 2 == 0 else args[-1] 1315 1316 def conform(x: Any, denan: Any = None, deinf: Any = None, fn = str) -> Any: 1317 'Make values JSON-compatible.' 1318 1319 if isinstance(x, float): 1320 # turn NaNs and Infinities into the replacement values given 1321 if isnan(x): 1322 return denan 1323 if isinf(x): 1324 return deinf 1325 return x 1326 1327 if isinstance(x, (bool, int, float, str)): 1328 return x 1329 1330 if isinstance(x, dict): 1331 return { 1332 str(k): conform(v) for k, v in x.items() if not 1333 (isinstance(k, Skip) or isinstance(v, Skip)) 1334 } 1335 1336 if isinstance(x, Iterable): 1337 return [conform(e) for e in x if not isinstance(e, Skip)] 1338 1339 if isinstance(x, DotCallable): 1340 return x.value 1341 1342 return fn(x) 1343 1344 fix = conform 1345 1346 def countif(src: Iterable, check: Callable) -> int: 1347 ''' 1348 Count how many values make the func given true-like. This func works with 1349 sequences, dictionaries, and strings. 1350 ''' 1351 1352 if callable(src): 1353 src, check = check, src 1354 check = predicate(check) 1355 1356 total = 0 1357 if isinstance(src, dict): 1358 for v in src.values(): 1359 if check(v): 1360 total += 1 1361 else: 1362 for v in src: 1363 if check(v): 1364 total += 1 1365 return total 1366 1367 # def debase64(x): 1368 # return debase64bytes(str(x).encode()).decode() 1369 1370 def debase64(s: str) -> bytes: 1371 'Convert away from base64 encoding, including data-URIs.' 1372 1373 if s.startswith('data:'): 1374 i = s.find(',') 1375 if i >= 0: 1376 return standard_b64decode(s[i + 1:]) 1377 return standard_b64decode(s) 1378 1379 unbase64 = debase64 1380 1381 def dedup(v: Iterable) -> Iterable: 1382 'Ignore reappearing items from iterables, after their first occurrence.' 1383 1384 got = set() 1385 for e in v: 1386 if not e in got: 1387 got.add(e) 1388 yield e 1389 1390 dedupe = dedup 1391 deduped = dedup 1392 deduplicate = dedup 1393 deduplicated = dedup 1394 undup = dedup 1395 undupe = dedup 1396 unduped = dedup 1397 unduplicate = dedup 1398 unduplicated = dedup 1399 unique = dedup 1400 uniqued = dedup 1401 1402 def defunc(x: Any) -> Any: 1403 'Call if value is a func, or return it back as given.' 1404 return x() if callable(x) else x 1405 1406 callmemaybe = defunc 1407 defunct = defunc 1408 unfunc = defunc 1409 unfunct = defunc 1410 1411 def dejson(x: Any, catch: Union[Callable[[Exception], Any], Any] = None) -> Any: 1412 'Safely parse JSON from strings.' 1413 try: 1414 return loads(x) if isinstance(x, str) else x 1415 except Exception as e: 1416 return catch(e) if callable(catch) else catch 1417 1418 unjson = dejson 1419 1420 def denan(x: Any, fallback: Any = None) -> Any: 1421 'Replace floating-point NaN with the alternative value given.' 1422 return x if not (isinstance(x, float) and isnan(x)) else fallback 1423 1424 def denil(*args: Any) -> Any: 1425 'Avoid None values, if possible: first value which isn\'t None wins.' 1426 for e in args: 1427 if e != None: 1428 return e 1429 return None 1430 1431 denone = denil 1432 denull = denil 1433 1434 def dirname(s: str) -> str: 1435 'Ignore the last part of a filepath.' 1436 return str(Path(s).parent) 1437 1438 def dive(into: Any, doing: Callable) -> Any: 1439 'Transform a nested value by calling a func via depth-first recursion.' 1440 1441 # support args in either order 1442 if callable(into): 1443 into, doing = doing, into 1444 1445 return _dive_kv(None, into, doing) 1446 1447 deepmap = dive 1448 dive1 = dive 1449 1450 def divebin(x: Any, y: Any, doing: Callable) -> Any: 1451 'Nested 2-value version of depth-first-recursive func dive.' 1452 1453 # support args in either order 1454 if callable(x): 1455 x, y, doing = y, doing, x 1456 1457 narg = required_arg_count(doing) 1458 if narg == 2: 1459 return dive(x, lambda a: dive(y, lambda b: doing(a, b))) 1460 if narg == 4: 1461 return dive(x, lambda i, a: dive(y, lambda j, b: doing(i, a, j, b))) 1462 raise Exception('divebin(...) only supports funcs with 2 or 4 args') 1463 1464 bindive = divebin 1465 # diveboth = divebin 1466 # dualdive = divebin 1467 # duodive = divebin 1468 dive2 = divebin 1469 1470 def _dive_kv(key: Any, into: Any, doing: Callable) -> Any: 1471 if isinstance(into, dict): 1472 return {k: _dive_kv(k, v, doing) for k, v in into.items()} 1473 if isinstance(into, Iterable) and not isinstance(into, str): 1474 return [_dive_kv(i, e, doing) for i, e in enumerate(into)] 1475 1476 narg = required_arg_count(doing) 1477 return doing(key, into) if narg == 2 else doing(into) 1478 1479 class DotCallable: 1480 'Enable convenient dot-syntax calling of 1-input funcs.' 1481 1482 def __init__(self, value: Any): 1483 self.value = value 1484 1485 def __getattr__(self, key: str) -> Any: 1486 return DotCallable(globals()[key](self.value)) 1487 1488 class Dottable: 1489 'Enable convenient dot-syntax access to dictionary values.' 1490 1491 def __getattr__(self, key: Any) -> Any: 1492 return self.__dict__[key] if key in self.__dict__ else None 1493 1494 def __getitem__(self, key: Any) -> Any: 1495 return self.__dict__[key] if key in self.__dict__ else None 1496 1497 def __iter__(self) -> Iterable: 1498 return iter(self.__dict__) 1499 1500 def dotate(x: Any) -> Union[Dottable, Any]: 1501 'Recursively ensure all dictionaries in a value are dot-accessible.' 1502 1503 if isinstance(x, dict): 1504 d = Dottable() 1505 d.__dict__ = {k: dotate(v) for k, v in x.items()} 1506 return d 1507 if isinstance(x, list): 1508 return [dotate(e) for e in x] 1509 if isinstance(x, tuple): 1510 return tuple(dotate(e) for e in x) 1511 return x 1512 1513 dotated = dotate 1514 dote = dotate 1515 doted = dotate 1516 dotified = dotate 1517 dotify = dotate 1518 dottified = dotate 1519 dottify = dotate 1520 1521 # make dictionaries `physics`, `symbols`, and `units` easier to use 1522 phy = dotate(phy) 1523 physics = phy 1524 sym = dotate(sym) 1525 symbols = sym 1526 units = dotate(units) 1527 1528 def drop(src: Any, *what) -> Any: 1529 ''' 1530 Either ignore all substrings occurrences, or ignore all keys given from 1531 an object, or even from a sequence of objects. 1532 ''' 1533 1534 if isinstance(src, str): 1535 return strdrop(src, *what) 1536 return _itemsdrop(src, set(what)) 1537 1538 dropped = drop 1539 # ignore = drop 1540 # ignored = drop 1541 1542 def _itemsdrop(src: Any, what: Set) -> Any: 1543 if isinstance(src, dict): 1544 kv = {} 1545 for k, v in src.items(): 1546 if not (k in what): 1547 kv[k] = v 1548 return kv 1549 1550 if isinstance(src, Iterable): 1551 return [_itemsdrop(e, what) for e in src] 1552 1553 return None 1554 1555 def each(src: Iterable, f: Callable) -> Any: 1556 ''' 1557 A generalization of built-in func map, which can also handle dictionaries 1558 and strings. 1559 ''' 1560 1561 if callable(src): 1562 src, f = f, src 1563 1564 if isinstance(src, dict): 1565 return mapkv(src, lambda k, _: k, f) 1566 1567 if isinstance(src, str): 1568 s = StringIO() 1569 f = loopify(f) 1570 for i, c in enumerate(src): 1571 v = f(i, c) 1572 if not isinstance(v, Skip): 1573 s.write(str(v)) 1574 return s.getvalue() 1575 1576 return tuple(f(i, v) for i, v in enumerate(src)) 1577 1578 mapped = each 1579 1580 def emptyish(x: Any) -> bool: 1581 ''' 1582 Check if a value can be considered empty, which includes non-empty 1583 strings which only have spaces in them. 1584 ''' 1585 1586 def check(x: Any) -> bool: 1587 if not x: 1588 return True 1589 if isinstance(x, str): 1590 return bool(emptyish_re.match(x)) 1591 return False 1592 1593 if check(x): 1594 return True 1595 if isinstance(x, Iterable): 1596 return all(check(e) for e in x) 1597 return False 1598 1599 isemptyish = emptyish 1600 1601 def endict(x: Any) -> Dict[str, Any]: 1602 'Turn non-dictionary values into dictionaries with string keys.' 1603 1604 if isinstance(x, dict): 1605 return {str(k): v for k, v in x.items()} 1606 if arrayish(x): 1607 return {str(e): e for e in x} 1608 return {str(x): x} 1609 1610 dicted = endict 1611 endicted = endict 1612 indict = endict 1613 todict = endict 1614 1615 def enfloat(x: Any, fallback: float = nan) -> float: 1616 try: 1617 return float(x) 1618 except Exception: 1619 return fallback 1620 1621 enfloated = enfloat 1622 floated = enfloat 1623 floatify = enfloat 1624 floatize = enfloat 1625 tofloat = enfloat 1626 1627 def enint(x: Any, fallback: Any = None) -> Any: 1628 try: 1629 return int(x) 1630 except Exception: 1631 return fallback 1632 1633 eninted = enint 1634 inted = enint 1635 integered = enint 1636 intify = enint 1637 intize = enint 1638 toint = enint 1639 1640 def enlist(x: Any) -> List[Any]: 1641 'Turn non-list values into lists.' 1642 return list(x) if arrayish(x) else [x] 1643 1644 # inlist = enlist 1645 enlisted = enlist 1646 listify = enlist 1647 listize = enlist 1648 tolist = enlist 1649 1650 def entuple(x: Any) -> Tuple[Any, ...]: 1651 'Turn non-tuple values into tuples.' 1652 return tuple(x) if arrayish(x) else (x, ) 1653 1654 entupled = entuple 1655 ntuple = entuple 1656 ntupled = entuple 1657 tuplify = entuple 1658 tuplize = entuple 1659 toentuple = entuple 1660 tontuple = entuple 1661 totuple = entuple 1662 1663 def error(message: Any) -> Exception: 1664 return Exception(str(message)) 1665 1666 err = error 1667 1668 def ext(s: str) -> str: 1669 'Get a filepath\'s extension, if present.' 1670 1671 name = Path(s).name 1672 i = name.rfind('.') 1673 return name[i:] if i >= 0 else '' 1674 1675 filext = ext 1676 1677 def fail(message: Any, error_code: int = 255) -> NoReturn: 1678 stdout.flush() 1679 print(f'\x1b[31m{message}\x1b[0m', file=stderr) 1680 quit(error_code) 1681 1682 abort = fail 1683 bail = fail 1684 1685 def fields(s: str) -> Iterable[str]: 1686 'Split fields AWK-style from the string given.' 1687 return awk_sep_re.split(s.strip()) 1688 1689 # items = fields 1690 splitfields = fields 1691 splititems = fields 1692 words = fields 1693 1694 def first(items: SupportsIndex, fallback: Any = None) -> Any: 1695 return items[0] if len(items) > 0 else fallback 1696 1697 def flappend(*args: Any) -> List[Any]: 1698 'Turn arbitrarily-nested values/sequences into a single flat sequence.' 1699 1700 flat = [] 1701 def dig(x: Any) -> None: 1702 if arrayish(x): 1703 for e in x: 1704 dig(e) 1705 elif isinstance(x, dict): 1706 for e in x.values(): 1707 dig(e) 1708 else: 1709 flat.append(x) 1710 1711 for e in args: 1712 dig(e) 1713 return flat 1714 1715 def flat(*args: Any) -> Iterable: 1716 'Turn arbitrarily-nested values/sequences into a single flat sequence.' 1717 1718 def _flat_rec(x: Any) -> Iterable: 1719 if x is None: 1720 return 1721 1722 if isinstance(x, dict): 1723 yield from _flat_rec(x.values()) 1724 1725 if isinstance(x, str): 1726 yield x 1727 return 1728 1729 if isinstance(x, Iterable): 1730 for e in x: 1731 yield from _flat_rec(e) 1732 return 1733 1734 yield x 1735 1736 for x in args: 1737 yield from _flat_rec(x) 1738 1739 flatten = flat 1740 flattened = flat 1741 1742 def fromto(start, stop, f: Callable = identity) -> Iterable: 1743 'Sequence all integers between the numbers given, end-value included.' 1744 return (f(e) for e in range(start, stop + 1)) 1745 1746 def fuzz(x: Union[int, float]) -> Union[float, Dict[str, float]]: 1747 ''' 1748 Deapproximate numbers to their max range before approximation: the 1749 result is a dictionary with the guessed lower-bound number, the number 1750 given, and the guessed upper-bound number which can approximate to the 1751 original number given. NaNs and the infinities are returned as given, 1752 instead of resulting in a dictionary. 1753 ''' 1754 1755 if isnan(x) or isinf(x): 1756 return x 1757 1758 if x == 0: 1759 return {'-0.5': -0.5, '0': 0.0, '0.5': +0.5} 1760 1761 if x % 1 != 0: 1762 # return surrounding integers when given non-integers 1763 a = floor(x) 1764 b = ceil(x) 1765 return {str(a): a, str(x): x, str(b): b} 1766 1767 if x % 10 != 0: 1768 a = x - 0.5 1769 b = x + 0.5 1770 return {str(a): a, str(x): x, str(b): b} 1771 1772 # find the integer log10 of the absolute value; 0 was handled previously 1773 y = int(abs(x)) 1774 p10 = 1 1775 while True: 1776 if y % p10 != 0: 1777 p10 /= 10 1778 break 1779 p10 *= 10 1780 delta = p10 / 2 1781 1782 s = +1 if x > 0 else -1 1783 ux = abs(x) 1784 a = s * ux - delta 1785 b = s * ux + delta 1786 return {str(a): a, str(x): x, str(b): b} 1787 1788 def generated(src: Any) -> Any: 1789 'Make tuples out of generators, or return non-generator values as given.' 1790 return tuple(src) if isinstance(src, (Generator, range)) else src 1791 1792 concrete = generated 1793 concreted = generated 1794 concretize = generated 1795 concretized = generated 1796 degen = generated 1797 degenerate = generated 1798 degenerated = generated 1799 degenerator = generated 1800 gen = generated 1801 generate = generated 1802 synth = generated 1803 synthed = generated 1804 synthesize = generated 1805 synthesized = generated 1806 1807 def group(src: Iterable, by: Callable = identity) -> Dict: 1808 ''' 1809 Separate transformed items into arrays, the final result being a dict 1810 whose keys are all the transformed values, and whose values are lists 1811 of all the original values which did transform to their group's key. 1812 ''' 1813 1814 if callable(src): 1815 src, by = by, src 1816 1817 by = loopify(by) 1818 kv = src.items() if isinstance(src, dict) else enumerate(src) 1819 1820 groups = {} 1821 for k, v in kv: 1822 dk = by(k, v) 1823 if isinstance(dk, Skip) or isinstance(v, Skip): 1824 continue 1825 if dk in groups: 1826 groups[dk].append(v) 1827 else: 1828 groups[dk] = [v] 1829 return groups 1830 1831 grouped = group 1832 1833 def gire(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict: 1834 ''' 1835 Group matched items into arrays, the final result being a dict whose 1836 keys are all the matchable regexes given, and whose values are lists 1837 of all the original values which did case-insensitively match their 1838 group's key as a regex. 1839 ''' 1840 1841 using = tuple(using) 1842 return group(src, lambda x: imatch(x, using, fallback)) 1843 1844 gbire = gire 1845 groupire = gire 1846 1847 def gre(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict: 1848 ''' 1849 Group matched items into arrays, the final result being a dict whose 1850 keys are all the matchable regexes given, and whose values are lists 1851 of all the original values which did regex-match their group's key. 1852 ''' 1853 1854 using = tuple(using) 1855 return group(src, lambda x: match(x, using, fallback)) 1856 1857 gbre = gre 1858 groupre = gre 1859 1860 def gsub(s: str, what: str, repl: str) -> str: 1861 'Replace all regex-matches with the string given.' 1862 return compile(what).sub(repl, s) 1863 1864 def harden(f: Callable, fallback: Any = None) -> Callable: 1865 def _hardened_caller(*args): 1866 try: 1867 return f(*args) 1868 except Exception: 1869 return fallback 1870 return _hardened_caller 1871 1872 hardened = harden 1873 insure = harden 1874 insured = harden 1875 1876 def horner(coeffs: List[float], x: Union[int, float]) -> float: 1877 if isinstance(coeffs, (int, float)): 1878 coeffs, x = x, coeffs 1879 1880 if len(coeffs) == 0: 1881 return 0 1882 1883 y = coeffs[0] 1884 for c in islice(coeffs, 1, None): 1885 y *= x 1886 y += c 1887 return y 1888 1889 polyval = horner 1890 1891 def idiota(n: int, f: Callable = identity) -> Dict[int, int]: 1892 'ID (keys) version of func iota.' 1893 return { v: v for v in (f(e) for e in range(1, n + 1))} 1894 1895 dictiota = idiota 1896 kviota = idiota 1897 1898 def imatch(what: str, using: Iterable[str], fallback: str = '') -> str: 1899 'Try to case-insensitively match a string with any of the regexes given.' 1900 1901 if not isinstance(what, str): 1902 what, using = using, what 1903 1904 for s in using: 1905 expr = compile(s, False) 1906 m = expr.search(what) 1907 if m: 1908 # return what[m.start():m.end()] 1909 return s 1910 return fallback 1911 1912 def indices(x: Any) -> Iterable[Any]: 1913 'List all indices/keys, or get an exclusive range from an int.' 1914 1915 if isinstance(x, int): 1916 return range(x) 1917 if isinstance(x, dict): 1918 return x.keys() 1919 if isinstance(x, (str, list, tuple)): 1920 return range(len(x)) 1921 return tuple() 1922 1923 keys = indices 1924 1925 def ints(start, stop, f: Callable = identity) -> Iterable[int]: 1926 'Sequence integers, end-value included.' 1927 1928 if isnan(start) or isnan(stop) or isinf(start) or isinf(stop): 1929 return tuple() 1930 return (f(e) for e in range(int(ceil(start)), int(stop) + 1)) 1931 1932 integers = ints 1933 1934 def iota(n: int, f: Callable = identity) -> Iterable[int]: 1935 'Sequence all integers from 1 up to (and including) the int given.' 1936 return (f(e) for e in range(1, n + 1)) 1937 1938 def itemsafter(x: Iterable, what: Any) -> Iterable: 1939 ok = False 1940 check = predicate(what) 1941 for e in x: 1942 if ok: 1943 yield e 1944 elif check(e): 1945 ok = True 1946 1947 def itemsafterlast(x: Iterable, what: Any) -> Iterable: 1948 rest: List[Any] = [] 1949 check = predicate(what) 1950 for e in x: 1951 if check(e): 1952 rest.clear() 1953 else: 1954 rest.append(e) 1955 1956 for e in islice(rest, 1, len(rest)): 1957 yield e 1958 1959 def itemsbefore(x: Iterable, what: Any) -> Iterable: 1960 check = predicate(what) 1961 for e in x: 1962 if check(e): 1963 return 1964 yield e 1965 1966 def itemsbeforelast(x: Iterable, what: Any) -> Iterable: 1967 items = [] 1968 for e in x: 1969 items.append(e) 1970 1971 i = -1 1972 check = predicate(what) 1973 for j, e in enumerate(reversed(items)): 1974 if check(e): 1975 i = j 1976 break 1977 1978 if i < 0: 1979 return items 1980 if i == 0: 1981 return tuple() 1982 for e in islice(items, 0, i): 1983 yield e 1984 1985 def itemssince(x: Iterable, what: Any) -> Iterable: 1986 ok = False 1987 check = predicate(what) 1988 for e in x: 1989 ok = ok or check(e) 1990 if ok: 1991 yield e 1992 1993 def itemssincelast(x: Iterable, what: Any) -> Iterable: 1994 rest: List[Any] = [] 1995 check = predicate(what) 1996 for e in x: 1997 if check(e): 1998 rest.clear() 1999 else: 2000 rest.append(e) 2001 return rest 2002 2003 def itemsuntil(x: Iterable, what: Any) -> Iterable: 2004 check = predicate(what) 2005 for e in x: 2006 yield e 2007 if check(e): 2008 return 2009 2010 def itemsuntillast(x: Iterable, what: Any) -> Iterable: 2011 items = [] 2012 for e in x: 2013 items.append(e) 2014 2015 i = -1 2016 check = predicate(what) 2017 for j, e in enumerate(reversed(items)): 2018 if check(e): 2019 i = j 2020 break 2021 2022 if i < 0: 2023 return items 2024 for e in islice(items, 0, i + 1): 2025 yield e 2026 2027 itemsuntilfinal = itemsuntillast 2028 2029 def join(items: Iterable, sep: Union[str, Iterable] = ' ') -> Union[str, Dict]: 2030 ''' 2031 Join iterables using the separator-string given: its 2 arguments 2032 can come in either order, and are sorted out if needed. When given 2033 2 non-string iterables, the result is an object whose keys are from 2034 the first argument, and whose values are from the second one. 2035 2036 You can use it any of the following ways, where `keys` and `values` are 2037 sequences (lists, tuples, or generators), and `separator` is a string: 2038 2039 join(values) 2040 join(values, separator) 2041 join(separator, values) 2042 join(keys, values) 2043 ''' 2044 2045 if arrayish(items) and arrayish(sep): 2046 return {k: v for k, v in zip(items, sep)} 2047 if isinstance(items, str): 2048 items, sep = sep, items 2049 return sep.join(str(e) for e in items) 2050 2051 def joined_paragraphs(lines: Iterable[str]) -> Iterable[Sequence[str]]: 2052 ''' 2053 Regroup lines into individual paragraphs, each of which can span multiple 2054 lines: such paragraphs have no empty lines in them, and never end with a 2055 trailing line-feed. 2056 ''' 2057 2058 par: List[str] = [] 2059 for l in lines: 2060 if (not l) and par: 2061 yield '\n'.join(par) 2062 par.clear() 2063 else: 2064 par.append(l) 2065 2066 if len(par) > 0: 2067 yield '\n'.join(par) 2068 2069 def json0(x: Any) -> str: 2070 'Encode value into a minimal single-line JSON string.' 2071 return dumps(x, separators=(',', ':'), allow_nan=False, indent=None) 2072 2073 j0 = json0 2074 2075 def json2(x: Any) -> str: 2076 ''' 2077 Encode value into a (possibly multiline) JSON string, using 2 spaces for 2078 each indentation level. 2079 ''' 2080 return dumps(x, separators=(',', ': '), allow_nan=False, indent=2) 2081 2082 j2 = json2 2083 2084 def jsonl(x: Any) -> Iterable: 2085 'Turn value into multiple JSON-encoded strings, known as JSON Lines.' 2086 2087 if x is None: 2088 yield dumps(x, allow_nan=False) 2089 elif isinstance(x, (bool, int, float, dict, str)): 2090 yield dumps(x, allow_nan=False) 2091 elif isinstance(x, Iterable): 2092 for e in x: 2093 yield dumps(e, allow_nan=False) 2094 else: 2095 yield dumps(str(x), allow_nan=False) 2096 2097 jsonlines = jsonl 2098 ndjson = jsonl 2099 tojsonl = jsonl 2100 tojsonlines = jsonl 2101 2102 def keep(src: Iterable, pred: Any) -> Iterable: 2103 ''' 2104 A generalization of built-in func filter, which can also handle dicts and 2105 strings. 2106 ''' 2107 2108 if callable(src): 2109 src, pred = pred, src 2110 pred = predicate(pred) 2111 pred = loopify(pred) 2112 2113 if isinstance(src, str): 2114 out = StringIO() 2115 for i, c in enumerate(src): 2116 if pred(i, c): 2117 out.write(c) 2118 return out.getvalue() 2119 2120 if isinstance(src, dict): 2121 return { k: v for k, v in src.items() if pred(k, v) } 2122 return (e for i, e in enumerate(src) if pred(i, e)) 2123 2124 filtered = keep 2125 kept = keep 2126 2127 def last(items: SupportsIndex, fallback: Any = None) -> Any: 2128 return items[-1] if len(items) > 0 else fallback 2129 2130 def links(src: Any) -> Iterable: 2131 'Auto-detect all (HTTP/HTTPS) hyperlink-like substrings.' 2132 2133 if isinstance(src, str): 2134 for match in link_re.finditer(src): 2135 # yield src[match.start():match.end()] 2136 yield match.group(0) 2137 elif isinstance(src, dict): 2138 for k, v in src.items(): 2139 yield from k 2140 yield from links(v) 2141 elif isinstance(src, Iterable): 2142 for v in src: 2143 yield from links(v) 2144 2145 def loopify(x: Callable) -> Callable: 2146 nargs = required_arg_count(x) 2147 if nargs == 2: 2148 return x 2149 elif nargs == 1: 2150 return lambda _, v: x(v) 2151 else: 2152 raise Exception('only funcs with 1 or 2 args are supported') 2153 2154 def mapkv(src: Iterable, key: Callable, value: Callable = identity) -> Dict: 2155 ''' 2156 A map-like func for dictionaries, which uses 2 mapping funcs, the first 2157 for the keys, the second for the values. 2158 ''' 2159 2160 if key is None: 2161 key = lambda k, _: k 2162 2163 if callable(src): 2164 src, key, value = value, src, key 2165 2166 if required_arg_count(key) != 2: 2167 oldkey = key 2168 key = lambda k, _: oldkey(k) 2169 2170 key = loopify(key) 2171 value = loopify(value) 2172 # if isinstance(src, dict): 2173 # return { key(k, v): value(k, v) for k, v in src.items() } 2174 # return { key(i, v): value(i, v) for i, v in enumerate(src) } 2175 2176 def add(k, v, to): 2177 dk = key(k, v) 2178 dv = value(k, v) 2179 if isinstance(dk, Skip) or isinstance(dv, Skip): 2180 return 2181 to[dk] = dv 2182 2183 res = {} 2184 kv = src.items() if isinstance(src, dict) else enumerate(src) 2185 for k, v in kv: 2186 add(k, v, res) 2187 return res 2188 2189 def match(what: str, using: Iterable[str], fallback: str = '') -> str: 2190 'Try to match a string with any of the regexes given.' 2191 2192 if not isinstance(what, str): 2193 what, using = using, what 2194 2195 for s in using: 2196 expr = compile(s) 2197 m = expr.search(what) 2198 if m: 2199 # return what[m.start():m.end()] 2200 return s 2201 return fallback 2202 2203 def maybe(f: Callable, x: Any) -> Any: 2204 ''' 2205 Try calling a func on a value, using the same value as a fallback result, 2206 in case of exceptions. 2207 ''' 2208 2209 if not callable(f): 2210 f, x = x, f 2211 try: 2212 return f(x) 2213 except Exception: 2214 return x 2215 2216 def mappend(*args) -> Dict: 2217 kv = {} 2218 for src in args: 2219 if isinstance(src, dict): 2220 for k, v in src.items(): 2221 kv[k] = v 2222 else: 2223 raise Exception('mappend only works with dictionaries') 2224 return kv 2225 2226 def message(x: Any, result: Any = skip) -> Any: 2227 print(x, file=stderr) 2228 return result 2229 2230 msg = message 2231 2232 def must(cond: Any, errmsg: str = 'condition given not always true') -> None: 2233 'Enforce conditions, raising an exception on failure.' 2234 if not cond: 2235 raise Exception(errmsg) 2236 2237 demand = must 2238 enforce = must 2239 2240 def nowdict() -> dict: 2241 v = datetime(2000, 1, 1).now() 2242 return { 2243 'year': v.year, 2244 'month': v.month, 2245 'day': v.day, 2246 'hour': v.hour, 2247 'minute': v.minute, 2248 'second': v.second, 2249 'text': v.strftime('%Y-%m-%d %H:%M:%S %b %a'), 2250 'weekday': v.strftime('%A'), 2251 } 2252 2253 def number(x: Any) -> Union[int, float, Any]: 2254 ''' 2255 Try to turn the value given into a number, using a fallback value instead 2256 of raising exceptions. 2257 ''' 2258 2259 if isinstance(x, float): 2260 return x 2261 2262 try: 2263 return int(x) 2264 except Exception: 2265 return float(x) 2266 2267 def numbers(src: Any) -> Iterable: 2268 'Auto-detect all number-like substrings.' 2269 2270 if isinstance(src, str): 2271 for match in number_re.finditer(src): 2272 yield match.group(0).strip() 2273 # yield src[match.start():match.end()].strip() 2274 elif isinstance(src, dict): 2275 for k, v in src.items(): 2276 yield from k 2277 yield from links(v) 2278 elif isinstance(src, Iterable): 2279 for v in src: 2280 yield from links(v) 2281 2282 def numsign(x: Union[int, float]) -> Union[int, float]: 2283 'Get a number\'s sign, or NaN if the number given is a NaN.' 2284 2285 if isinstance(x, int): 2286 if x > 0: 2287 return +1 2288 if x < 0: 2289 return -1 2290 return 0 2291 2292 if isnan(x): 2293 return x 2294 2295 if x > 0: 2296 return +1.0 2297 if x < 0: 2298 return -1.0 2299 return 0.0 2300 2301 def numstats(src: Any) -> Dict[str, Union[float, int]]: 2302 'Gather several single-pass numeric statistics.' 2303 2304 n = mean_sq = ln_sum = 0 2305 least = +inf 2306 most = -inf 2307 total = mean = 0 2308 prod = 1 2309 nans = ints = pos = zero = neg = 0 2310 2311 def update_numstats(x: Any) -> None: 2312 nonlocal nans, n, ints, pos, neg, zero, least, most, total, prod 2313 nonlocal ln_sum, mean, mean_sq 2314 2315 if not isinstance(x, (float, int)): 2316 return 2317 2318 if isnan(x): 2319 nans += 1 2320 return 2321 2322 n += 1 2323 ints += int(isinstance(x, int) or x == floor(x)) 2324 2325 if x > 0: 2326 pos += 1 2327 elif x < 0: 2328 neg += 1 2329 else: 2330 zero += 1 2331 2332 least = min(least, x) 2333 most = max(most, x) 2334 2335 # total += x 2336 prod *= x 2337 ln_sum += log(x) 2338 2339 d1 = x - mean 2340 mean += d1 / n 2341 d2 = x - mean 2342 mean_sq += d1 * d2 2343 2344 def _numstats_rec(src: Any) -> None: 2345 if isinstance(src, dict): 2346 for e in src.values(): 2347 _numstats_rec(e) 2348 elif isinstance(src, Iterable) and not isinstance(src, str): 2349 for e in src: 2350 _numstats_rec(e) 2351 else: 2352 update_numstats(src) 2353 2354 _numstats_rec(src) 2355 2356 sd = nan 2357 geomean = nan 2358 if n > 0: 2359 sd = sqrt(mean_sq / n) 2360 geomean = exp(ln_sum / n) if not isinf(ln_sum) else nan 2361 total = n * mean 2362 2363 return { 2364 'n': n, 2365 'nan': nans, 2366 'min': least, 2367 'max': most, 2368 'sum': total, 2369 'mean': mean, 2370 'geomean': geomean, 2371 'sd': sd, 2372 'product': prod, 2373 'integer': ints, 2374 'positive': pos, 2375 'zero': zero, 2376 'negative': neg, 2377 } 2378 2379 def once(x: Any, replacement: Any = None) -> Any: 2380 ''' 2381 Replace the first argument given after the first time this func has been 2382 given it: this is a deliberately stateful function, given its purpose. 2383 ''' 2384 2385 if not (x in seen): 2386 seen.add(x) 2387 return x 2388 else: 2389 return replacement 2390 2391 onced = once 2392 2393 def pad(s: str, n: int, pad: str = ' ') -> str: 2394 l = len(s) 2395 return s if l >= n else s + int((n - l) / len(pad)) * pad 2396 2397 def padcenter(s: str, n: int, pad: str = ' ') -> str: 2398 return s.center(n, pad) 2399 2400 centerpad = padcenter 2401 centerpadded = padcenter 2402 cjust = padcenter 2403 cpad = padcenter 2404 padc = padcenter 2405 paddedcenter = padcenter 2406 2407 def padend(s: str, n: int, pad: str = ' ') -> str: 2408 return s.rjust(n, pad) 2409 2410 padr = padend 2411 padright = padend 2412 paddedend = padend 2413 paddedright = padend 2414 rpad = padend 2415 rightpad = padend 2416 rightpadded = padend 2417 2418 def padstart(s: str, n: int, pad: str = ' ') -> str: 2419 return s.ljust(n, pad) 2420 2421 lpad = padstart 2422 leftpad = padstart 2423 leftpadded = padstart 2424 padl = padstart 2425 padleft = padstart 2426 paddedleft = padstart 2427 paddedstart = padstart 2428 2429 def panic(x: Any) -> None: 2430 raise Exception(x) 2431 2432 def paragraphize(lines: Iterable[str]) -> Iterable[Sequence[str]]: 2433 ''' 2434 Regroup lines into individual paragraphs, each of which is a list of 2435 single-line strings, none of which never end with a trailing line-feed. 2436 ''' 2437 2438 par: List[str] = [] 2439 for l in lines: 2440 if (not l) and par: 2441 yield par 2442 par.clear() 2443 else: 2444 par.append(l) 2445 2446 if len(par) > 0: 2447 yield par 2448 2449 paragraphed = paragraphize 2450 paragraphs = paragraphize 2451 paragroup = paragraphize 2452 pargroup = paragraphize 2453 2454 def parse(s: str, fallback: Any = None) -> Any: 2455 'Try to parse JSON, ignoring exceptions in favor of a fallback value.' 2456 2457 try: 2458 return loads(s) 2459 except Exception: 2460 return fallback 2461 2462 fromjson = parse 2463 parsed = parse 2464 loaded = parse 2465 unjson = parse 2466 2467 def pick(src: Any, *what) -> Any: 2468 'Pick only the keys given from an object, or even a sequence of objects.' 2469 2470 if isinstance(src, dict): 2471 kv = {} 2472 for k in what: 2473 kv[k] = src[k] 2474 return kv 2475 2476 if isinstance(src, Iterable): 2477 return [pick(e, *what) for e in src] 2478 2479 return None 2480 2481 picked = pick 2482 2483 def plain(s: str) -> str: 2484 'Ignore all ANSI-style sequences in a string.' 2485 return ansi_style_re.sub('', s) 2486 2487 def predicate(x: Any) -> Callable: 2488 'Helps various higher-order funcs, by standardizing `predicate` values.' 2489 2490 if callable(x): 2491 return x 2492 2493 if isinstance(x, float): 2494 if isnan(x): 2495 return lambda y: isinstance(y, float) and isnan(y) 2496 if isinf(x): 2497 return lambda y: isinstance(y, float) and isinf(y) 2498 2499 return lambda y: x == y 2500 2501 pred = predicate 2502 2503 def quoted(s: str, quote: str = '"') -> str: 2504 'Surround a string with quotes.' 2505 return f'{quote}{s}{quote}' 2506 2507 def recover(*args) -> Any: 2508 ''' 2509 Catch exceptions using a lambda/callback func, in one of 6 ways 2510 recover(zero_args_func) 2511 recover(zero_args_func, exception_replacement_value) 2512 recover(zero_args_func, one_arg_exception_handling_func) 2513 recover(one_arg_func, arg) 2514 recover(one_arg_func, arg, exception_replacement_value) 2515 recover(one_arg_func, arg, one_arg_exception_handling_func) 2516 ''' 2517 2518 if len(args) == 1: 2519 f = args[0] 2520 try: 2521 return f() 2522 except Exception: 2523 return None 2524 elif len(args) == 2: 2525 f, fallback = args[0], args[1] 2526 if callable(f) and callable(fallback): 2527 try: 2528 return f() 2529 except Exception as e: 2530 nargs = required_arg_count(fallback) 2531 return fallback(e) if nargs == 1 else fallback() 2532 else: 2533 try: 2534 return f() if required_arg_count(f) == 0 else f(args[1]) 2535 except Exception: 2536 return fallback 2537 elif len(args) == 3: 2538 f, x, fallback = args[0], args[1], args[2] 2539 if callable(f) and callable(fallback): 2540 try: 2541 return f(x) 2542 except Exception as e: 2543 nargs = required_arg_count(fallback) 2544 return fallback(e) if nargs == 1 else fallback() 2545 else: 2546 try: 2547 return f(x) 2548 except Exception: 2549 return fallback 2550 else: 2551 raise Exception('recover(...) only works with 1, 2, or 3 args') 2552 2553 attempt = recover 2554 attempted = recover 2555 recovered = recover 2556 recoverred = recover 2557 rescue = recover 2558 rescued = recover 2559 trycall = recover 2560 2561 def reject(src: Iterable, pred: Any) -> Iterable: 2562 ''' 2563 A generalization of built-in func filter, which uses predicate funcs the 2564 opposite way, and which can also handle dicts and strings. 2565 ''' 2566 2567 if callable(src): 2568 src, pred = pred, src 2569 pred = predicate(pred) 2570 pred = loopify(pred) 2571 2572 if isinstance(src, str): 2573 out = StringIO() 2574 for i, c in enumerate(src): 2575 if not pred(i, c): 2576 out.write(c) 2577 return out.getvalue() 2578 2579 if isinstance(src, dict): 2580 return { k: v for k, v in src.items() if not pred(k, v) } 2581 return (e for i, e in enumerate(src) if not pred(i, e)) 2582 2583 avoid = reject 2584 avoided = reject 2585 keepout = reject 2586 keptout = reject 2587 rejected = reject 2588 2589 def retype(x: Any) -> Any: 2590 'Try to narrow the type of the value given.' 2591 2592 if isinstance(x, float): 2593 return int(x) if floor(x) == x else x 2594 2595 if not isinstance(x, str): 2596 return x 2597 2598 try: 2599 return loads(x) 2600 except Exception: 2601 pass 2602 2603 try: 2604 return int(x) 2605 except Exception: 2606 pass 2607 2608 try: 2609 return float(x) 2610 except Exception: 2611 pass 2612 2613 return x 2614 2615 autocast = retype 2616 mold = retype 2617 molded = retype 2618 narrow = retype 2619 narrowed = retype 2620 recast = retype 2621 recasted = retype 2622 remold = retype 2623 remolded = retype 2624 retyped = retype 2625 2626 def revcompose(*what: Callable) -> Callable: 2627 def composite(x: Any) -> Any: 2628 for f in reversed(what): 2629 x = f(x) 2630 return x 2631 return composite 2632 2633 rcompose = revcompose 2634 rcomposed = revcompose 2635 revcomposed = revcompose 2636 2637 def revsort(iterable: Iterable, key: Optional[Callable] = None) -> Iterable: 2638 return sorted(iterable, key=key, reverse=True) 2639 2640 revsorted = revsort 2641 2642 # def revsortkv(src: Dict, key: Callable = None) -> Dict: 2643 # if not key: 2644 # key = lambda kv: (kv[1], kv[0]) 2645 # return sortkv(src, key, reverse=True) 2646 2647 def revsortkv(src: Dict, key: Callable = None) -> Dict: 2648 if key is None: 2649 key = lambda x: x[1] 2650 return sortkv(src, key, reverse=True) 2651 2652 revsortedkv = revsortkv 2653 2654 def rstripdecs(s: str) -> str: 2655 ''' 2656 Ignore trailing zero decimals on number-like strings; even ignore 2657 the decimal dot if trailing. 2658 ''' 2659 2660 try: 2661 f = float(s) 2662 if isnan(f) or isinf(f): 2663 return s 2664 2665 dot = s.find('.') 2666 if dot < 0: 2667 return s 2668 2669 s = s.rstrip('0') 2670 return s[:-1] if s.endswith('.') else s 2671 except Exception: 2672 return s 2673 2674 chopdecs = rstripdecs 2675 2676 def scale(x: float, x0: float, x1: float, y0: float, y1: float) -> float: 2677 'Transform a value from a linear domain into another linear one.' 2678 return (y1 - y0) * (x - x0) / (x1 - x0) + y0 2679 2680 rescale = scale 2681 rescaled = scale 2682 scaled = scale 2683 2684 def shortened(s: str, maxlen: int, trailer: str = '') -> str: 2685 'Limit strings to the symbol-count given, including an optional trailer.' 2686 maxlen = max(maxlen, 0) 2687 return s if len(s) <= maxlen else s[:maxlen - len(trailer)] + trailer 2688 2689 def shuffled(x: Any) -> Any: 2690 'Return a shuffled copy of the list given.' 2691 y = copy(x) 2692 shuffle(y) 2693 return y 2694 2695 def split(src: Union[str, Sequence], n: Union[str, int]) -> Iterable: 2696 'Split/break a string/sequence into several chunks/parts.' 2697 2698 if isinstance(src, str) and isinstance(n, str): 2699 return src.split(n) 2700 if not (isinstance(src, (str, Sequence)) and isinstance(n, int)): 2701 raise Exception('unsupported type-pair of arguments') 2702 2703 if n < 1: 2704 return [] 2705 2706 l = len(src) 2707 if l <= n: 2708 return src.split('') if isinstance(src, str) else src 2709 2710 chunks = [] 2711 csize = int(ceil(l / n)) 2712 while len(src) > 0: 2713 chunks.append(src[:csize]) 2714 src = src[csize:] 2715 return chunks 2716 2717 broken = split 2718 splitted = split 2719 splitten = split 2720 2721 def strdrop(x: str, *what: str) -> str: 2722 'Ignore all occurrences of all substrings given.' 2723 2724 for s in what: 2725 x = x.replace(s, '') 2726 return x 2727 2728 strignore = strdrop 2729 2730 def stringify(x: Any) -> str: 2731 'Fancy alias for func dumps, named after JavaScript\'s func.' 2732 return dumps(x, separators=(', ', ': '), allow_nan=False, indent=None) 2733 2734 jsonate = stringify 2735 jsonify = stringify 2736 tojson = stringify 2737 2738 def strafter(x: str, what: str) -> str: 2739 i = x.find(what) 2740 return '' if i < 0 else x[i+len(what):] 2741 2742 def strafterlast(x: str, what: str) -> str: 2743 i = x.rfind(what) 2744 return '' if i < 0 else x[i+len(what):] 2745 2746 def strbefore(x: str, what: str) -> str: 2747 i = x.find(what) 2748 return x if i < 0 else x[:i] 2749 2750 def strbeforelast(x: str, what: str) -> str: 2751 i = x.rfind(what) 2752 return x if i < 0 else x[:i] 2753 2754 def strsince(x: str, what: str) -> str: 2755 i = x.find(what) 2756 return '' if i < 0 else x[i:] 2757 2758 def strsincelast(x: str, what: str) -> str: 2759 i = x.rfind(what) 2760 return '' if i < 0 else x[i:] 2761 2762 def struntil(x: str, what: str) -> str: 2763 i = x.find(what) 2764 return x if i < 0 else x[:i+len(what)] 2765 2766 def struntillast(x: str, what: str) -> str: 2767 i = x.rfind(what) 2768 return x if i < 0 else x[:i+len(what)] 2769 2770 struntilfinal = struntillast 2771 2772 def since(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2773 'Start strings/sequences with a substring/value\'s appearance.' 2774 return (strsince if isinstance(x, str) else itemssince)(x, what) 2775 2776 def sincelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2777 'Start strings/sequences with a substring/value\'s last appearance.' 2778 return (strsincelast if isinstance(x, str) else itemssincelast)(x, what) 2779 2780 sincefinal = sincelast 2781 2782 def sortk(x: Dict, key: Callable = identity, reverse: bool = False) -> Dict: 2783 keys = sorted(x.keys(), key=key, reverse=reverse) 2784 return {k: x[k] for k in keys} 2785 2786 sortkeys = sortk 2787 sortedkeys = sortk 2788 2789 def sortkv(src: Dict, key: Callable = None, reverse: bool = False) -> Dict: 2790 if key is None: 2791 key = lambda x: x[1] 2792 kv = sorted(src.items(), key=key, reverse=reverse) 2793 return {k: v for (k, v) in kv} 2794 2795 sortedkv = sortkv 2796 2797 def squeeze(s: str) -> str: 2798 ''' 2799 A more aggressive way to rid strings of extra spaces which, unlike string 2800 method strip, also turns inner runs of multiple spaces into single ones. 2801 ''' 2802 s = s.strip() 2803 s = spaces_re.sub(' ', s) 2804 s = paddable_tab_re.sub('\t', s) 2805 return s 2806 2807 squeezed = squeeze 2808 2809 def stround(x: Union[int, float], decimals: int = 6) -> str: 2810 'Format numbers into a string with the given decimal-digit count.' 2811 2812 if decimals >= 0: 2813 return f'{x:.{decimals}f}' 2814 else: 2815 return f'{round(x, decimals):.0f}' 2816 2817 def tally(src: Iterable, by: Callable = identity) -> Dict[Any, int]: 2818 ''' 2819 Count all distinct (transformed) values, the result being a dictionary 2820 whose keys are all the transformed values, and whose items are positive 2821 integers. 2822 ''' 2823 2824 if callable(src): 2825 src, by = by, src 2826 2827 tally: Dict[Any, int] = {} 2828 by = loopify(by) 2829 2830 if isinstance(src, dict): 2831 for k, v in src.items(): 2832 dk = by(k, v) 2833 if dk in tally: 2834 tally[dk] += 1 2835 else: 2836 tally[dk] = 1 2837 else: 2838 for i, v in enumerate(src): 2839 dk = by(i, v) 2840 if dk in tally: 2841 tally[dk] += 1 2842 else: 2843 tally[dk] = 1 2844 return tally 2845 2846 tallied = tally 2847 2848 def transpose(src: Any) -> Any: 2849 'Turn lists/objects inside-out like socks, so to speak.' 2850 2851 if isinstance(src, dict): 2852 return { v: k for k, v in src.items() } 2853 2854 if not arrayish(src): 2855 msg = 'transpose only supports objects or iterables of objects' 2856 raise ValueError(msg) 2857 2858 kv: Dict[Any, Any] = {} 2859 seq: List[Any] = [] 2860 2861 for e in src: 2862 if isinstance(e, dict): 2863 for k, v in e.items(): 2864 if k in kv: 2865 kv[k].append(v) 2866 else: 2867 kv[k] = [v] 2868 elif isinstance(e, Iterable): 2869 for i, v in enumerate(e): 2870 if i < len(seq): 2871 seq[i].append(v) 2872 else: 2873 seq.append([v]) 2874 else: 2875 msg = 'transpose(...): not all items are iterables/objects' 2876 raise ValueError(msg) 2877 2878 if len(kv) > 0 and len(seq) > 0: 2879 msg = 'transpose(...): mix of iterables and objects not supported' 2880 raise ValueError(msg) 2881 return kv if len(seq) == 0 else seq 2882 2883 tr = transpose 2884 transp = transpose 2885 transposed = transpose 2886 2887 def trap(x: Callable, y: Union[Callable[[Exception], Any], Any] = None) -> Any: 2888 'Try running a func, handing exceptions over to a fallback func.' 2889 2890 try: 2891 return x() if callable(x) else x 2892 except Exception as e: 2893 if callable(y): 2894 nargs = required_arg_count(y) 2895 return y(e) if nargs == 1 else y() 2896 else: 2897 return y 2898 2899 catch = trap 2900 catched = trap 2901 caught = trap 2902 noerr = trap 2903 noerror = trap 2904 noerrors = trap 2905 safe = trap 2906 save = trap 2907 saved = trap 2908 trapped = trap 2909 2910 def tsv(x: str, fn: Union[Callable, None] = None) -> Any: 2911 if fn is None: 2912 return x.split('\t') 2913 if callable(x): 2914 x, fn = fn, x 2915 return fn(x.split('\t')) 2916 2917 def typename(x: Any) -> str: 2918 if x is None: 2919 return 'null' 2920 if isinstance(x, bool): 2921 return 'boolean' 2922 if isinstance(x, str): 2923 return 'string' 2924 if isinstance(x, (int, float)): 2925 return 'number' 2926 if isinstance(x, (list, tuple)): 2927 return 'array' 2928 if isinstance(x, dict): 2929 return 'object' 2930 return type(x).__name__ 2931 2932 def typeof(x: Any) -> str: 2933 'Get a value\'s JS-like typeof type-string.' 2934 2935 if callable(x): 2936 return 'function' 2937 2938 return { 2939 bool: 'boolean', 2940 int: 'number', 2941 float: 'number', 2942 str: 'string', 2943 }.get(type(x), 'object') 2944 2945 def unixify(s: str) -> str: 2946 ''' 2947 Make plain-text `unix-style`, ignoring a leading UTF-8 BOM if present, 2948 and turning any/all CRLF byte-pairs into line-feed bytes. 2949 ''' 2950 s = s.lstrip('\xef\xbb\xbf') 2951 return s.replace('\r\n', '\n') if '\r\n' in s else s 2952 2953 def unquoted(s: str) -> str: 2954 'Ignore surrounding quotes in a string.' 2955 2956 if s.startswith('"') and s.endswith('"'): 2957 return s[1:-1] 2958 if s.startswith('\'') and s.endswith('\''): 2959 return s[1:-1] 2960 if s.startswith('`') and s.endswith('`'): 2961 return s[1:-1] 2962 if s.startswith('”') and s.endswith('“'): 2963 return s[1:-1] 2964 if s.startswith('“') and s.endswith('”'): 2965 return s[1:-1] 2966 return s 2967 2968 dequote = unquoted 2969 dequoted = unquoted 2970 2971 def until(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2972 'End strings/sequences with a substring/value\'s appearance.' 2973 return (struntil if isinstance(x, str) else itemsuntil)(x, what) 2974 2975 def untillast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]: 2976 'End strings/sequences with a substring/value\'s last appearance.' 2977 return (struntillast if isinstance(x, str) else itemsuntillast)(x, what) 2978 2979 untilfinal = untillast 2980 2981 2982 def wait(seconds: Union[int, float], result: Any) -> Any: 2983 'Wait the given number of seconds, before returning its latter arg.' 2984 2985 t = (int, float) 2986 if (not isinstance(seconds, t)) and isinstance(result, t): 2987 seconds, result = result, seconds 2988 sleep(seconds) 2989 return result 2990 2991 delay = wait 2992 2993 def wat(*args) -> None: 2994 'What Are These (wat) shows help/doc messages for funcs given to it.' 2995 2996 from pydoc import doc 2997 2998 c = 0 2999 w = stderr 3000 3001 for e in args: 3002 if not callable(e): 3003 continue 3004 3005 if c > 0: 3006 print(file=w) 3007 3008 print(f'\x1b[48;5;253m\x1b[38;5;26m{e.__name__:80}\x1b[0m', file=w) 3009 doc(e, output=w) 3010 c += 1 3011 3012 return Skip() 3013 3014 def wit(*args) -> None: 3015 'What Is This (wit) shows help/doc messages for funcs given to it.' 3016 return wat(*args) 3017 3018 def zoom(x: Any, *keys_indices) -> Any: 3019 for k in keys_indices: 3020 # allow int-indexing dicts the same way lists/tuples can be 3021 if isinstance(x, dict) and isinstance(k, int): 3022 l = len(x) 3023 if i < 0: 3024 i += l 3025 if i < 0 or i >= len(x): 3026 x = None 3027 continue 3028 for i, e in enumerate(x.values()): 3029 if i == k: 3030 x = e 3031 break 3032 continue 3033 3034 # regular key/index access for dicts/lists/tuples 3035 x = x[k] 3036 3037 return x 3038 3039 3040 no_input_opts = ( 3041 '=', '-None', '--None', '-nil', '--nil', '-noinput', '--noinput', 3042 '-no-input', '--no-input', '-none', '--none', '-null', '--null', 3043 '-null-input', '--null-input', '-nullinput', '--nullinput', '--n', 3044 ) 3045 json0_opts = ( 3046 '-c', '--c', '-compact', '--compact', '-j0', '--j0', 3047 '-json0', '--json0', '-json-0', '--json-0', 3048 ) 3049 traceback_opts = ( 3050 '-t', '--t', '-trace', '--trace', '-traceback', '--traceback', 3051 ) 3052 profile_opts = ('-p', '--p', '-prof', '--prof', '-profile', '--profile') 3053 dot_opts = ('-d', '--d', '-dot', '--dot', '-dots', '--dots') 3054 zoom_opts = ('-z', '--z', '-zj', '--zj', '-zoom', '--zoom') 3055 3056 3057 # no args or a leading help-option arg means show the help message and quit 3058 if len(argv) == 3: 3059 if argv[1] in ('-h', '--h', '-help', '--help') and argv[2] in zoom_opts: 3060 print(zj_info_msg.strip(), file=stderr) 3061 exit(0) 3062 3063 3064 args = argv[1:] 3065 load_input = True 3066 dot_input = False 3067 trace_exceptions = False 3068 profile_eval = False 3069 compact_output = False 3070 expression = None 3071 name = '' 3072 3073 # handle all other leading options; the explicit help options are 3074 # handled earlier in the script 3075 while len(args) > 0: 3076 if args[0] in no_input_opts: 3077 load_input = False 3078 args = args[1:] 3079 elif args[0] in json0_opts: 3080 compact_output = True 3081 args = args[1:] 3082 elif args[0] in traceback_opts: 3083 trace_exceptions = True 3084 args = args[1:] 3085 elif args[0] in profile_opts: 3086 profile_eval = True 3087 args = args[1:] 3088 elif args[0] in dot_opts: 3089 dot_input = True 3090 args = args[1:] 3091 elif args[0] in zoom_opts: 3092 # if len(args) < 2: 3093 # print(zj_info_msg.strip(), file=stderr) 3094 # exit(0) 3095 3096 try: 3097 z = load(stdin) 3098 z = zj_zoom(z, tuple(args[1:])) 3099 ind = None if compact_output else 2 3100 seps = (',', ':') if compact_output else (',', ': ') 3101 dump(z, stdout, indent=ind, separators=seps, allow_nan=False) 3102 stdout.write('\n') 3103 except BrokenPipeError: 3104 # quit quietly, instead of showing a confusing error message 3105 stderr.close() 3106 except KeyboardInterrupt: 3107 exit(2) 3108 except Exception as e: 3109 if trace_exceptions: 3110 raise e 3111 s = str(e) 3112 s = s if s else '<generic exception>' 3113 print(f'\x1b[31m{s}\x1b[0m', file=stderr) 3114 exit(1) 3115 3116 exit(0) 3117 else: 3118 break 3119 3120 if len(args) > 2 and load_input: 3121 print('\x1b[31mmultiple inputs not allowed\x1b[0m', file=stderr) 3122 exit(1) 3123 3124 if len(args) == 1: 3125 expression = args[0] 3126 args = args[1:] 3127 elif len(args) > 1: 3128 expression = args[0] 3129 name = args[1] 3130 args = args[1:] 3131 3132 if expression is None: 3133 print(info.strip(), file=stderr) 3134 exit(0) 3135 3136 glo = globals() 3137 for e in (physics, symbols, units): 3138 for _k, _v in e.__dict__.items(): 3139 if not _k in glo: 3140 glo[_k] = _v 3141 3142 try: 3143 # load JSON into variable `v`, unless input was explicitly disabled 3144 v = None 3145 if load_input: 3146 try: 3147 if name == '' or name == '-': 3148 v = load(stdin) 3149 elif seems_url(name): 3150 from urllib.request import urlopen 3151 with urlopen(name) as inp: 3152 v = load(inp) 3153 else: 3154 with open(name, encoding='utf-8') as inp: 3155 v = load(inp) 3156 except Exception as e: 3157 raise Exception(f'JSON-input error: {e}') 3158 3159 if dot_input: 3160 v = dotate(v) 3161 3162 # offer several aliases for main variable `v`; the intuitive 3163 # `in` (short for `input`) is a keyword, so it's not available 3164 data = value = d = dat = val = v 3165 3166 # transform data using the formula/expression given, handling 3167 # single dots as identity operations 3168 exec = disabled_exec 3169 open = disabled_open 3170 if not expression or expression == '.': 3171 expression = 'data' 3172 expression = compile_py(expression, expression, 'eval') 3173 if profile_eval: 3174 from cProfile import Profile 3175 # using a profiler in a `with` context adds many irrelevant 3176 # entries to its output 3177 prof = Profile() 3178 prof.enable() 3179 v = eval(expression) 3180 if result_needs_fixing(v): 3181 v = fix_result(v, data) 3182 prof.disable() 3183 prof.print_stats() 3184 else: 3185 v = eval(expression) 3186 if result_needs_fixing(v): 3187 v = fix_result(v, data) 3188 3189 # emit result as JSON 3190 try: 3191 ind = None if compact_output else 2 3192 seps = (',', ':') if compact_output else (',', ': ') 3193 dump(v, stdout, indent=ind, separators=seps, allow_nan=False) 3194 stdout.write('\n') 3195 except Exception as e: 3196 raise Exception(f'JSON-output error: {e}') 3197 except BrokenPipeError: 3198 # quit quietly, instead of showing a confusing error message 3199 stderr.close() 3200 except KeyboardInterrupt: 3201 exit(2) 3202 except Exception as e: 3203 if trace_exceptions: 3204 raise e 3205 s = str(e) 3206 s = s if s else '<generic exception>' 3207 print(f'\x1b[31m{s}\x1b[0m', file=stderr) 3208 exit(1)