File: tl.py
   1 #!/usr/bin/python
   2 
   3 # The MIT License (MIT)
   4 #
   5 # Copyright (c) 2026 pacman64
   6 #
   7 # Permission is hereby granted, free of charge, to any person obtaining a copy
   8 # of this software and associated documentation files (the "Software"), to deal
   9 # in the Software without restriction, including without limitation the rights
  10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11 # copies of the Software, and to permit persons to whom the Software is
  12 # furnished to do so, subject to the following conditions:
  13 #
  14 # The above copyright notice and this permission notice shall be included in
  15 # all copies or substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23 # SOFTWARE.
  24 
  25 
  26 info = '''
  27 tl [options...] [python expression] [filepaths/URIs...]
  28 
  29 
  30 Transform Lines runs a Python expression on each line of plain-text data:
  31 each expression given emits its result as its own line. Each input line is
  32 available to the expression as either `line`, or `l`. Lines are always
  33 stripped of any trailing end-of-line bytes/sequences.
  34 
  35 When the expression results in non-string iterable values, a sort of input
  36 `amplification` happens for the current input-line, where each item from
  37 the result is emitted on its own output line. Dictionaries emit their data
  38 as a single JSON line.
  39 
  40 When a formula's result is the None value, it emits no output line, which
  41 filters-out the current line, the same way empty-iterable results do.
  42 
  43 When in `all` mode, all input lines are read first into a list of strings,
  44 whose items are all stripped of any end-of-line sequences, and kept in the
  45 `lines` global variable: the expression given is then run only once.
  46 
  47 Similarly, if the argument before the expression is a single equals sign
  48 (a `=`, but without the quotes), no data are read/loaded: the expression is
  49 then run only once, effectively acting as a `pure` plain-text generator.
  50 
  51 Current-input names, depending on mode:
  52 
  53     names               mode                 evaluation
  54 
  55     l, line             each-line (default)  for each input line
  56     lines               all-lines            once, after last input line
  57     b, block, p, par    block/paragraph      for each paragraph of lines
  58     v, value            jsonl                for each input line
  59     (no name)           no-input             once, without an input value
  60 
  61 Modes `each-line` (the default) and `block/paragraph` also define `i` as an
  62 integer which starts as 0, and which is incremented after each evaluation.
  63 
  64 Options, where leading double-dashes are also allowed, except for alias `=`:
  65 
  66     -a           read all lines at once into a string-list called `lines`
  67     -all         same as -a
  68     -lines       same as -a
  69 
  70     -b           read uninterrupted blocks/groups of lines as paragraphs
  71     -blocks      same as -b
  72     -g           same as -b
  73     -groups      same as -b
  74     -p           same as -b
  75     -par         same as -b
  76     -para        same as -b
  77     -paragraphs  same as -b
  78 
  79     -h           show this help message
  80     -help        same as -h
  81 
  82     -jsonl       transform JSON Lines into proper JSON
  83 
  84     -nil         don't read any input, and run the expression only once
  85     -no-input    same as -nil
  86     -noinput     same as -nil
  87     -none        same as -nil
  88     -null        same as -nil
  89     -null-input  same as -nil
  90     -nullinput   same as -nil
  91     =            same as -nil
  92 
  93     -p           show a performance/time-profile of the full `task` run
  94     -prof        same as -p
  95     -profile     same as -p
  96 
  97     -s           read each input as a whole string
  98     -str         same as -s
  99     -string      same as -s
 100     -w           same as -s
 101     -whole       same as -s
 102 
 103     -t           show a full traceback of this script for exceptions
 104     -trace       same as -t
 105     -traceback   same as -t
 106 
 107 
 108 Extra Functions
 109 
 110 blue(s)           color strings blue, using surrounding ANSI-style sequences
 111 gray(s)           color strings gray, using surrounding ANSI-style sequences
 112 green(s)          color strings green, using surrounding ANSI-style sequences
 113 highlight(s)      highlight strings, using surrounding ANSI-style sequences
 114 hilite(s)         same as func highlight
 115 orange(s)         color strings orange, using surrounding ANSI-style sequences
 116 purple(s)         color strings purple, using surrounding ANSI-style sequences
 117 red(s)            color strings red, using surrounding ANSI-style sequences
 118 
 119 realign(x, gap=2) pad items across lines, so that all "columns" align
 120 
 121 after(x, y)       ignore items until the one given; for strings and sequences
 122 afterfinal(x, y)  backward counterpart of func after
 123 afterlast(x, y)   same as func afterfinal
 124 arrayish(x)       check if value is a list, a tuple, or a generator
 125 basename(s)       get the final/file part of a pathname
 126 before(x, y)      ignore items since the one given; for strings and sequences
 127 beforefinal(x, y) backward counterpart of func before
 128 beforelast(x, y)  same as func beforefinal
 129 chunk(x, size)    split/resequence items into chunks of the length given
 130 chunked(x, size)  same as func chunk
 131 compose(*args)    make a func which chain-calls all funcs given
 132 composed(*args)   same as func compose
 133 cond(*args)       expression-friendly fully-evaluated if-else chain
 134 debase64(s)       decode base64 strings, including data-URIs
 135 dedup(x)          ignore later (re)occurrences of values in a sequence
 136 dejson(x, f=None) safe parse JSON from strings
 137 denan(x, y)       turn a floating-point NaN values into the fallback given
 138 denil(*args)      return the first non-null/none value among those given
 139 denone(*args)     same as func denil
 140 denull(*args)     same as func denil
 141 dirname(s)        get the folder/directory/parent part of a pathname
 142 dive(x, f)        transform value in depth-first-recursive fashion
 143 divebin(x, y, f)  binary (2-input) version of recursive-transform func dive
 144 drop(x, *what)    ignore keys or substrings; for strings, dicts, dict-lists
 145 dropped(x, *v)    same as func drop
 146 each(x, f)        generalization of built-in func map
 147 endict(x)         turn non-dictionary values into dicts with string keys
 148 enfloat(x, f=nan) turn values into floats, offering a fallback on failure
 149 enint(x, f=None)  turn values into ints, offering a fallback on failure
 150 enlist(x)         turn non-list values into lists
 151 entuple(x)        turn non-tuple values into tuples
 152 ext(s)            return the file-extension part of a pathname, if available
 153 fields(s)         split fields AWK-style from the string given
 154 filtered(x, f)    same as func keep
 155 flat(*args)       flatten everything into an unnested sequence
 156 fromto(x, y, ?f)  sequence integers, end-value included
 157 group(x, ?by)     group values into dicts of lists; optional transform func
 158 grouped(x, ?by)   same as func group
 159 harden(f, v)      make funcs which return values instead of exceptions
 160 hardened(f, v)    same as func harden
 161 countif(x, f)     count how many values make the func given true-like
 162 idiota(x, ?f)     dict-counterpart of func iota
 163 ints(x, y, ?f)    make sequences of increasing integers, which include the end
 164 iota(x, ?f)       make an integer sequence from 1 up to the number given
 165 join(x, y)        join values into a string; make a dict from keys and values
 166 json0(x)          turn a value into its smallest JSON-string representation
 167 json2(x)          turn a value into a 2-space-indented multi-line JSON string
 168 jsonl(x)          turn a value into a sequence of single-line (JSONL) strings
 169 keep(x, pred)     generalization of built-in func filter
 170 kept(x, pred)     same as func keep
 171 links(x)          auto-detect all hyperlink-like (HTTP/HTTPS) substrings
 172 mapped(x, f)      same as func each
 173 number(x)         try to parse as an int, on failure try to parse as a float
 174 numbers(x)        auto-detect all numbers in the value given
 175 numstats(x)       calculate various `single-pass` numeric stats
 176 once(x, y=None)   avoid returning the same value more than once; stateful func
 177 pick(x, *what)    keep only the keys given; works on dicts, or dict-sequences
 178 picked(x, *what)  same a func pick
 179 plain(s)          ignore ANSI-style sequences in strings
 180 quoted(s, q='"')  surround a string with the (optional) quoting-symbol given
 181 recover(*args)    recover from exceptions with a fallback value
 182 reject(x, pred)   generalization of built-in func filter, with opposite logic
 183 since(x, y)       ignore items before the one given; for strings and sequences
 184 sincefinal(x, y)  backward counterpart of func since
 185 sincelast(x, y)   same as func sincefinal
 186 split(x, y)       split string by separator; split sequence into several ones
 187 squeeze(s)        strip/trim a string, squishing inner runs of spaces
 188 stround(x, d=6)   format numbers into decimal-number strings
 189 tally(x, ?by)     count/tally values, using an optional transformation func
 190 tallied(x, ?by)   same as func tally
 191 trap(x, f=None)   try running a func, handing exceptions to a fallback func
 192 trycall(*args)    same as func recover
 193 unique(x)         same as func dedup
 194 uniqued(x)        same as func dedup
 195 unjson(x, f=None) same as func dejson
 196 unquoted(s)       ignore surrounding quotes, if present
 197 until(x, y)       ignore items after the one given; for strings and sequences
 198 untilfinal(x, y)  backward counterpart of func until
 199 untillast(x, y)   same as func untilfinal
 200 wait(seconds, x)  wait the given number of seconds, before returning a value
 201 wat(*args)        What Are These (wat) shows help/doc messages for funcs
 202 
 203 
 204 Examples
 205 
 206 # numbers from 0 to 5, each on its own output line; no input is read/used
 207 tl = 'range(6)'
 208 
 209 # all powers up to the 4th, using each input line auto-parsed into a `float`
 210 tl = 'range(1, 6)' | tl '(float(l)**p for p in range(1, 4+1))'
 211 
 212 # separate input lines with an empty line between each; global var `empty`
 213 # can be used to avoid bothering with nested shell-quoting
 214 tl = 'range(6)' | tl '["", l] if i > 0 else l'
 215 
 216 # keep only the last 2 lines from the input
 217 tl = 'range(1, 6)' | tl -all 'lines[-2:]'
 218 
 219 # join input lines into tab-separated lines of up to 3 items each; global
 220 # var named `tab` can be used to avoid bothering with nested shell-quoting
 221 tl = 'range(1, 8)' | tl -all '("\\t".join(c) for c in chunk(lines, 3))'
 222 
 223 # ignore all lines before the first one with just a '5' in it
 224 tl = 'range(8)' | tl -all 'since(lines, "5")'
 225 
 226 # ignore errors/exceptions, in favor of the original lines/values
 227 tl = '("abc", "123")' | tl 'safe(lambda: 2 * float(line), line)'
 228 
 229 # ignore errors/exceptions, calling a fallback func with the exception
 230 tl = '("abc", "123")' | tl 'safe(lambda: 2 * float(line), lambda e: str(e))'
 231 
 232 # filtering lines out via None values
 233 head -c 1024 /dev/urandom | strings | tl 'l if len(l) < 20 else None'
 234 
 235 # boolean-valued results are concise ways to filter lines out
 236 head -c 1024 /dev/urandom | strings | tl 'len(l) < 20'
 237 
 238 # function/callable results are automatically called on the current line
 239 head -c 1024 /dev/urandom | strings | tl len
 240 '''
 241 
 242 
 243 from sys import argv, exit, stderr, stdin, stdout
 244 
 245 
 246 if __name__ != '__main__':
 247     print('don\'t import this script, run it directly instead', file=stderr)
 248     exit(1)
 249 
 250 # no args or a leading help-option arg means show the help message and quit
 251 if len(argv) < 2 or argv[1] in ('-h', '--h', '-help', '--help'):
 252     from sys import exit, stderr
 253     print(info.strip(), file=stderr)
 254     exit(0)
 255 
 256 
 257 from io import StringIO, TextIOWrapper
 258 
 259 from typing import \
 260     AbstractSet, Annotated, Any, AnyStr, \
 261     AsyncContextManager, AsyncGenerator, AsyncIterable, AsyncIterator, \
 262     Awaitable, BinaryIO, ByteString, Callable, cast, \
 263     ClassVar, Collection, Container, \
 264     ContextManager, Coroutine, Deque, Dict, Final, \
 265     final, ForwardRef, FrozenSet, Generator, Generic, get_args, get_origin, \
 266     get_type_hints, Hashable, IO, ItemsView, \
 267     Iterable, Iterator, KeysView, List, Literal, Mapping, \
 268     MappingView, Match, MutableMapping, MutableSequence, MutableSet, \
 269     NamedTuple, NewType, no_type_check, no_type_check_decorator, \
 270     NoReturn, Optional, overload, \
 271     Protocol, Reversible, \
 272     runtime_checkable, Sequence, Set, Sized, SupportsAbs, \
 273     SupportsBytes, SupportsComplex, SupportsFloat, SupportsIndex, \
 274     SupportsInt, SupportsRound, Text, TextIO, Tuple, Type, \
 275     TypedDict, TypeVar, \
 276     TYPE_CHECKING, Union, ValuesView
 277 try:
 278     from typing import \
 279         assert_never, assert_type, clear_overloads, Concatenate, \
 280         dataclass_transform, get_overloads, is_typeddict, LiteralString, \
 281         Never, NotRequired, ParamSpec, ParamSpecArgs, ParamSpecKwargs, \
 282         Required, reveal_type, Self, TypeAlias, TypeGuard, TypeVarTuple, \
 283         Unpack
 284     from typing import \
 285         AwaitableGenerator, override, TypeAliasType, type_check_only
 286 except Exception:
 287     pass
 288 
 289 
 290 def conforms(x: Any) -> bool:
 291     '''
 292     Check if a value is JSON-compatible, which includes checking values
 293     recursively, in case of composite/nestable values.
 294     '''
 295 
 296     if x is None or isinstance(x, (bool, int, str)):
 297         return True
 298     if isinstance(x, float):
 299         return not (isnan(x) or isinf(x))
 300     if isinstance(x, (list, tuple)):
 301         return all(conforms(e) for e in x)
 302     if isinstance(x, dict):
 303         return all(conforms(k) and conforms(v) for k, v in x.items())
 304     return False
 305 
 306 
 307 def seems_url(s: str) -> bool:
 308     protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:')
 309     return any(s.startswith(p) for p in protocols)
 310 
 311 
 312 def disabled_exec(*args, **kwargs) -> None:
 313     _ = args
 314     _ = kwargs
 315     raise Exception('built-in func `exec` is disabled')
 316 
 317 
 318 def fix_value(x: Any, default: Any) -> Any:
 319     'Adapt a value so it can be output.'
 320 
 321     # true shows the current line as the current output; presumably
 322     # this is the result of calling a `condition-like` expression
 323     if x is True:
 324         return default
 325 
 326     # null and false show no output for the current input line
 327     if x is False:
 328         return None
 329 
 330     if x is type:
 331         return type(default).__name__
 332 
 333     # if expression results in a func, auto-call it with the original data
 334     if callable(x) and not isinstance(x, Iterable):
 335         c = required_arg_count(x)
 336         if c == 1:
 337             x = x(default)
 338         else:
 339             m = f'func auto-call only works with 1-arg funcs (func wanted {c})'
 340             raise Exception(m)
 341 
 342     if x is None or isinstance(x, (bool, int, float, str)):
 343         return x
 344 
 345     rec = fix_value
 346 
 347     if isinstance(x, dict):
 348         return {
 349             rec(k, default): rec(v, default) for k, v in x.items() if not
 350                 (isinstance(k, Skip) or isinstance(v, Skip))
 351         }
 352     if isinstance(x, Iterable):
 353         return tuple(rec(e, default) for e in x if not isinstance(e, Skip))
 354 
 355     if isinstance(x, Dottable):
 356         return rec(x.__dict__, default)
 357     if isinstance(x, DotCallable):
 358         return rec(x.value, default)
 359 
 360     if isinstance(x, Exception):
 361         raise x
 362 
 363     return None if isinstance(x, Skip) else str(x)
 364 
 365 
 366 def show_value(w, x: Any) -> None:
 367     'Helper func used by func show_result.'
 368 
 369     # null shows no output for the current input line
 370     if x is None or isinstance(x, Skip):
 371         return
 372 
 373     if isinstance(x, dict):
 374         dump(x, w, separators=(', ', ': '), allow_nan=False, indent=None)
 375         w.write('\n')
 376         w.flush()
 377     elif isinstance(x, (bytes, bytearray)):
 378         w.write(x)
 379         w.flush()
 380     elif isinstance(x, Iterable) and not isinstance(x, str):
 381         dump(x, w, separators=(', ', ': '), allow_nan=False, indent=None)
 382         w.write('\n')
 383         w.flush()
 384     elif isinstance(x, DotCallable):
 385         print(x.value, file=w, flush=True)
 386     else:
 387         print(x, file=w, flush=True)
 388 
 389 
 390 def show_result(w, x: Any) -> None:
 391     if isinstance(x, (dict, str)):
 392         show_value(w, x)
 393     elif isinstance(x, Iterable):
 394         for e in x:
 395             if isinstance(e, Exception):
 396                 raise e
 397             show_value(w, e)
 398     else:
 399         show_value(w, x)
 400 
 401 
 402 def make_open_utf8(open: Callable) -> Callable:
 403     'Restrict the file-open func to a read-only utf-8 file-open func.'
 404     def open_utf8_readonly(name: str):
 405         'A UTF-8 read-only file-open func overriding the built-in open func.'
 406         return open(name, encoding='utf-8')
 407     return open_utf8_readonly
 408 
 409 open_utf8 = make_open_utf8(open)
 410 open = open_utf8
 411 
 412 
 413 def loop_lines_inputs(r, inputs: List[str], doing: Callable) -> None:
 414     '''
 415     Act on multiple named inputs line-by-line, via the func given; when
 416     not given any named inputs, the default reader given is used instead.
 417     '''
 418 
 419     main_input: List[str] = []
 420     got_main_input = False
 421     dashes = inputs.count('-')
 422 
 423     if any(seems_url(e) for e in inputs):
 424         from urllib.request import urlopen
 425 
 426     def _adapt_lines(src) -> Iterable[str]:
 427         for j, line in enumerate(src):
 428             if j == 0:
 429                 line = line.lstrip('\xef\xbb\xbf')
 430             yield line.rstrip('\r\n').rstrip('\n')
 431 
 432     def _hold_adapt_lines(src) -> Iterable[str]:
 433         for e in _adapt_lines(src):
 434             main_input.append(e)
 435             yield e
 436 
 437     for path in inputs:
 438         if path == '-':
 439             if dashes == 1:
 440                 doing(_adapt_lines(r))
 441                 continue
 442 
 443             if not got_main_input:
 444                 doing(_hold_adapt_lines(r))
 445                 got_main_input = True
 446             else:
 447                 doing(_adapt_lines(main_input))
 448             continue
 449 
 450         if seems_url(path):
 451             with urlopen(path) as inp:
 452                 with TextIOWrapper(inp, encoding='utf-8') as txt:
 453                     doing(_adapt_lines(txt))
 454             continue
 455 
 456         with open_utf8(path) as inp:
 457             doing(_adapt_lines(inp))
 458 
 459     if len(inputs) == 0:
 460         doing(_adapt_lines(r))
 461 
 462 
 463 def loop_whole_inputs(r, inputs: List[str], doing: Callable) -> None:
 464     '''
 465     Act on multiple named inputs, read as whole strings, via the func given;
 466     when not given any named inputs, the default reader given is used instead.
 467     '''
 468 
 469     main_input: List[str] = []
 470     got_main_input = False
 471     dashes = inputs.count('-')
 472 
 473     if any(seems_url(e) for e in inputs):
 474         from urllib.request import urlopen
 475 
 476     for path in inputs:
 477         if path == '-':
 478             if dashes == 1:
 479                 doing(r.read())
 480                 continue
 481 
 482             if not got_main_input:
 483                 main_input = r.read()
 484                 got_main_input = True
 485             doing(main_input)
 486             continue
 487 
 488         if seems_url(path):
 489             with urlopen(path) as inp:
 490                 with TextIOWrapper(inp, encoding='utf-8') as txt:
 491                     doing(txt.read())
 492             continue
 493 
 494         with open_utf8(path) as inp:
 495             doing(inp.read())
 496 
 497     if len(inputs) == 0:
 498         doing(r.read())
 499 
 500 
 501 def main_whole_strings(out, r, expression, inputs) -> None:
 502     def _each_string(out, src, expression: Any) -> None:
 503         # `comprehension` expressions seem to ignore local variables: even
 504         # lambda-based workarounds fail
 505         global s, t, text, v, value, w, whole, _
 506 
 507         s = t = text = v = value = w = whole = src
 508         res = eval(expression)
 509         res = fix_value(res, src)
 510         show_result(out, res)
 511         _ = res
 512 
 513     loop_whole_inputs(r, inputs, lambda s: _each_string(out, s, expression))
 514 
 515 
 516 def main_each_line(w, r, expression, inputs) -> None:
 517     def _each_line(w, src, expression: Any) -> None:
 518         # `comprehension` expressions seem to ignore local variables: even
 519         # lambda-based workarounds fail
 520         global i, nr, previous, prev, line, l, line, _
 521 
 522         previous = ''
 523         prev = previous
 524 
 525         for line in src:
 526             l = line
 527             res = eval(expression)
 528             res = fix_value(res, line)
 529             show_result(w, res)
 530             i += 1
 531             nr += 1
 532             previous = line
 533             prev = previous
 534             _ = res
 535 
 536     loop_lines_inputs(r, inputs, lambda r: _each_line(w, r, expression))
 537 
 538 
 539 def main_each_block(w, r, expression, inputs) -> None:
 540     def _each_block(w, r, expression) -> None:
 541         # `comprehension` expressions seem to ignore local variables: even
 542         # lambda-based workarounds fail
 543         global i, nr
 544         global previous, prev, lines, block, par, para, paragraph, _
 545 
 546         for item in paragraphize(r):
 547             lines = block = par = para = paragraph = item
 548             res = eval(expression)
 549             if isinstance(res, Skip):
 550                 previous = data
 551                 prev = previous
 552                 i += 1
 553                 nr += 1
 554                 continue
 555 
 556             res = fix_value(res, lines)
 557             show_result(w, res)
 558             i += 1
 559             nr += 1
 560             prev = previous = lines
 561             _ = res
 562 
 563     loop_lines_inputs(r, inputs, lambda r: _each_block(w, r, expression))
 564 
 565 
 566 def main_all_lines(w, r, expression, inputs) -> None:
 567     # `comprehension` expressions seem to ignore local variables: even
 568     # lambda-based workarounds fail
 569     global line, lines, data, values, d, l, v, dat, val
 570 
 571     def _all_lines(w, r, expression) -> None:
 572         # `comprehension` expressions seem to ignore local variables: even
 573         # lambda-based workarounds fail
 574         global lines, line, l
 575 
 576         for e in r:
 577             line = l = e
 578             lines.append(line)
 579 
 580     lines = []
 581     line = l = ''
 582     loop_lines_inputs(r, inputs, lambda r: _all_lines(w, r, expression))
 583     data = values = d = v = dat = val = lines
 584     res = eval(expression)
 585     res = fix_value(res, lines)
 586     show_result(w, res)
 587 
 588 
 589 def main_all_bytes(w, r, expression, inputs) -> None:
 590     # `comprehension` expressions seem to ignore local variables: even
 591     # lambda-based workarounds fail
 592     global data, values, d, v, dat, val
 593     data = values = d = v = dat = val = r.buffer.read()
 594     res = eval(expression)
 595     res = fix_value(res, data)
 596     show_result(w, res)
 597 
 598 
 599 def main_no_input(w, r, expression, inputs) -> None:
 600     res = eval(expression)
 601     fix = lambda x: fix_value(x, None)
 602     f = str if res is None or isinstance(res, bool) else fix
 603     res = f(res)
 604     show_result(w, res)
 605 
 606 
 607 def main_json_lines(w, r, expression, inputs) -> None:
 608     def _jsonl2json(w, src, expression: Any) -> None:
 609         # `comprehension` expressions seem to ignore local variables: even
 610         # lambda-based workarounds fail
 611         global i, nr
 612         global line, l, data, d, value, v, dat, val, prev, previous, _
 613 
 614         previous = None
 615         prev = previous
 616 
 617         for line in src:
 618             if emptyish_re.match(line) or commented_re.match(line):
 619                 continue
 620             l = line
 621 
 622             data = value = d = v = dat = val = loads(line)
 623             res = eval(expression)
 624 
 625             if isinstance(res, Skip):
 626                 previous = data
 627                 prev = previous
 628                 i += 1
 629                 nr += 1
 630                 continue
 631             res = fix_value(res, data)
 632 
 633             if callable(res):
 634                 res = res(data)
 635             if not conforms(res):
 636                 res = conform(res)
 637             dump(res, w)
 638             _ = res
 639             w.write('\n')
 640 
 641             previous = data
 642             prev = previous
 643             i += 1
 644             nr += 1
 645 
 646     loop_lines_inputs(r, inputs, lambda r: _jsonl2json(w, r, expression))
 647 
 648 
 649 # opts2modes simplifies option-handling in func main
 650 opts2modes = {
 651     '=': 'no-input',
 652     '-nil': 'no-input',
 653     '-no-input': 'no-input',
 654     '-noinput': 'no-input',
 655     '-none': 'no-input',
 656     '-None': 'no-input',
 657     '-null': 'no-input',
 658     '-null-input': 'no-input',
 659     '-nullinput': 'no-input',
 660     '--n': 'no-input',
 661     '--nil': 'no-input',
 662     '--no-input': 'no-input',
 663     '--noinput': 'no-input',
 664     '--none': 'no-input',
 665     '--None': 'no-input',
 666     '--null': 'no-input',
 667     '--null-input': 'no-input',
 668     '--nullinput': 'no-input',
 669     '-a': 'all-lines',
 670     '-all': 'all-lines',
 671     '-lines': 'all-lines',
 672     '--a': 'all-lines',
 673     '--all': 'all-lines',
 674     '--lines': 'all-lines',
 675     '-b': 'each-block',
 676     '-blocks': 'each-block',
 677     '-g': 'each-block',
 678     '-groups': 'each-block',
 679     '-p': 'each-block',
 680     '-par': 'each-block',
 681     '-para': 'each-block',
 682     '-paragraphs': 'each-block',
 683     '--b': 'each-block',
 684     '--blocks': 'each-block',
 685     '--g': 'each-block',
 686     '--groups': 'each-block',
 687     '--p': 'each-block',
 688     '--par': 'each-block',
 689     '--para': 'each-block',
 690     '--paragraphs': 'each-block',
 691     '-bytes': 'bytes',
 692     '--bytes': 'bytes',
 693     '-jl': 'json-lines',
 694     '-jsonl': 'json-lines',
 695     '-jsonlines': 'json-lines',
 696     '-json-lines': 'json-lines',
 697     '--jl': 'json-lines',
 698     '--jsonl': 'json-lines',
 699     '--jsonlines': 'json-lines',
 700     '--json-lines': 'json-lines',
 701     '-s': 'whole-strings',
 702     '-str': 'whole-strings',
 703     '-string': 'whole-strings',
 704     '--s': 'whole-strings',
 705     '--str': 'whole-strings',
 706     '--string': 'whole-strings',
 707     '-w': 'whole-strings',
 708     '-whole': 'whole-strings',
 709     '--w': 'whole-strings',
 710     '--whole': 'whole-strings',
 711 }
 712 
 713 
 714 def blue(s: Any) -> str:
 715     'Blue-style a plain string via ANSI-style sequences.'
 716     return f'\x1b[38;5;26m{s}\x1b[0m'
 717 
 718 def blueback(s: Any) -> str:
 719     'Blue-background-style a plain string via ANSI-style sequences.'
 720     return f'\x1b[48;5;26m\x1b[38;5;255m{s}\x1b[0m'
 721 
 722 bluebg = blueback
 723 
 724 def bold(s: Any) -> str:
 725     'Bold-style a plain string via ANSI-style sequences.'
 726     return f'\x1b[1m{s}\x1b[0m'
 727 
 728 def gbm(s: str, good: Any = False, bad: Any = False, meh: Any = False) -> str:
 729     '''
 730     Good, Bad, Meh ANSI-styles a plain string via ANSI-style sequences,
 731     according to 1..3 conditions given as boolean(ish) values: these are
 732     checked in order, so the first truish one wins.
 733     '''
 734 
 735     if good:
 736         return green(s)
 737     if bad:
 738         return red(s)
 739     if meh:
 740         return gray(s)
 741     return s
 742 
 743 def gray(s: Any) -> str:
 744     'Gray-style a plain string via ANSI-style sequences.'
 745     return f'\x1b[38;5;248m{s}\x1b[0m'
 746 
 747 def grayback(s: Any) -> str:
 748     'Gray-background-style a plain string via ANSI-style sequences.'
 749     return f'\x1b[48;5;253m{s}\x1b[0m'
 750 
 751 graybg = grayback
 752 
 753 def green(s: Any) -> str:
 754     'Green-style a plain string via ANSI-style sequences.'
 755     return f'\x1b[38;5;29m{s}\x1b[0m'
 756 
 757 def greenback(s: Any) -> str:
 758     'Green-background-style a plain string via ANSI-style sequences.'
 759     return f'\x1b[48;5;29m\x1b[38;5;255m{s}\x1b[0m'
 760 
 761 greenbg = greenback
 762 
 763 def highlight(s: Any) -> str:
 764     'Highlight/reverse-style a plain string via ANSI-style sequences.'
 765     return f'\x1b[7m{s}\x1b[0m'
 766 
 767 hilite = highlight
 768 
 769 def magenta(s: Any) -> str:
 770     'Magenta-style a plain string via ANSI-style sequences.'
 771     return f'\x1b[38;5;165m{s}\x1b[0m'
 772 
 773 def magentaback(s: Any) -> str:
 774     'Magenta-background-style a plain string via ANSI-style sequences.'
 775     return f'\x1b[48;5;165m\x1b[38;5;255m{s}\x1b[0m'
 776 
 777 magback = magentaback
 778 magbg = magentaback
 779 magentabg = magentaback
 780 
 781 def orange(s: Any) -> str:
 782     'Orange-style a plain string via ANSI-style sequences.'
 783     return f'\x1b[38;5;166m{s}\x1b[0m'
 784 
 785 def orangeback(s: Any) -> str:
 786     'Orange-background-style a plain string via ANSI-style sequences.'
 787     return f'\x1b[48;5;166m\x1b[38;5;255m{s}\x1b[0m'
 788 
 789 orangebg = orangeback
 790 orback = orangeback
 791 orbg = orangeback
 792 
 793 def purple(s: Any) -> str:
 794     'Purple-style a plain string via ANSI-style sequences.'
 795     return f'\x1b[38;5;99m{s}\x1b[0m'
 796 
 797 def purpleback(s: Any) -> str:
 798     'Purple-background-style a plain string via ANSI-style sequences.'
 799     return f'\x1b[48;5;99m\x1b[38;5;255m{s}\x1b[0m'
 800 
 801 purback = purpleback
 802 purbg = purpleback
 803 purplebg = purpleback
 804 
 805 def red(s: Any) -> str:
 806     'Red-style a plain string via ANSI-style sequences.'
 807     return f'\x1b[38;5;1m{s}\x1b[0m'
 808 
 809 def redback(s: Any) -> str:
 810     'Red-background-style a plain string via ANSI-style sequences.'
 811     return f'\x1b[48;5;1m\x1b[38;5;255m{s}\x1b[0m'
 812 
 813 redbg = redback
 814 
 815 def underline(s: Any) -> str:
 816     'Underline-style a plain string via ANSI-style sequences.'
 817     return f'\x1b[4m{s}\x1b[0m'
 818 
 819 # def blue(s):
 820 #     return f'\x1b[38;2;0;95;215m{s}\x1b[0m'
 821 
 822 # def blueback(s):
 823 #     return f'\x1b[48;2;0;95;215m\x1b[38;2;238;238;238m{s}\x1b[0m'
 824 
 825 # bluebg = blueback
 826 
 827 # def bold(s):
 828 #     return f'\x1b[1m{s}\x1b[0m'
 829 
 830 # bolded = bold
 831 
 832 # def gray(s):
 833 #     return f'\x1b[38;2;168;168;168m{s}\x1b[0m'
 834 
 835 # def grayback(s):
 836 #     return f'\x1b[48;2;168;168;168m\x1b[38;2;238;238;238m{s}\x1b[0m'
 837 
 838 # def green(s):
 839 #     return f'\x1b[38;2;0;135;95m{s}\x1b[0m'
 840 
 841 # def greenback(s):
 842 #     return f'\x1b[48;2;0;135;95m\x1b[38;2;238;238;238m{s}\x1b[0m'
 843 
 844 # def highlight(s):
 845 #     return f'\x1b[7m{s}\x1b[0m'
 846 
 847 # hilite = highlight
 848 
 849 # def orange(s):
 850 #     return f'\x1b[38;2;215;95;0m{s}\x1b[0m'
 851 
 852 # def orangeback(s):
 853 #     return f'\x1b[48;2;215;95;0m\x1b[38;2;238;238;238m{s}\x1b[0m'
 854 
 855 # def purple(s):
 856 #     return f'\x1b[38;2;135;95;255m{s}\x1b[0m'
 857 
 858 # def purpleback(s):
 859 #     return f'\x1b[48;2;135;95;255m\x1b[38;2;238;238;238m{s}\x1b[0m'
 860 
 861 # def red(s):
 862 #     return f'\x1b[38;2;204;0;0m{s}\x1b[0m'
 863 
 864 # def redback(s):
 865 #     return f'\x1b[38;2;204;0;0m\x1b[38;2;238;238;238m{s}\x1b[0m'
 866 
 867 # def underline(s):
 868 #     return f'\x1b[4m{s}\x1b[0m'
 869 
 870 # underlined = underline
 871 
 872 
 873 def realign(lines: List[str], gap: int = 2) -> Iterable:
 874     '''
 875     Pad lines so that their items align across/vertically: extra padding
 876     is put between such `columns`, using 2 spaces by default.
 877     '''
 878 
 879     widths: List[int] = []
 880     for l in lines:
 881         items = awk_sep_re.split(l.strip())
 882         while len(widths) < len(items):
 883             widths.append(0)
 884         for i, s in enumerate(items):
 885             widths[i] = max(widths[i], len(s))
 886 
 887     sb = StringIO()
 888     gap = max(gap, 0)
 889 
 890     for l in lines:
 891         sb.truncate(0)
 892         sb.seek(0)
 893 
 894         padding = 0
 895         items = awk_sep_re.split(l.strip())
 896         for s, w in zip(items, widths):
 897             sb.write(padding * ' ')
 898             sb.write(s)
 899             padding = max(w - len(s), 0) + gap
 900 
 901         yield sb.getvalue()
 902 
 903 
 904 def stop_normal(x: Any, exit_code: int = 0) -> NoReturn:
 905     show_result(stdout, fix_value(x, None))
 906     exit(exit_code)
 907 
 908 
 909 def stop_json(x: Any, exit_code: int = 0) -> NoReturn:
 910     dump(x, stdout)
 911     stdout.write('\n')
 912     stdout.flush()
 913     exit(exit_code)
 914 
 915 
 916 from base64 import \
 917     standard_b64encode, standard_b64decode, \
 918     standard_b64encode as base64bytes, standard_b64decode as debase64bytes
 919 
 920 from collections import \
 921     ChainMap, Counter, defaultdict, deque, namedtuple, OrderedDict, \
 922     UserDict, UserList, UserString
 923 
 924 from copy import copy, deepcopy
 925 
 926 from datetime import \
 927     MAXYEAR, MINYEAR, date, datetime, time, timedelta, timezone, tzinfo
 928 try:
 929     from datetime import now, UTC
 930 except Exception:
 931     now = lambda: datetime(2000, 1, 1).now()
 932 
 933 from decimal import Decimal, getcontext
 934 
 935 from difflib import \
 936     context_diff, diff_bytes, Differ, get_close_matches, HtmlDiff, \
 937     IS_CHARACTER_JUNK, IS_LINE_JUNK, ndiff, restore, SequenceMatcher, \
 938     unified_diff
 939 
 940 from fractions import Fraction
 941 
 942 import functools
 943 from functools import \
 944     cache, cached_property, cmp_to_key, get_cache_token, lru_cache, \
 945     namedtuple, partial, partialmethod, recursive_repr, reduce, \
 946     singledispatch, singledispatchmethod, total_ordering, update_wrapper, \
 947     wraps
 948 
 949 from glob import glob, iglob
 950 
 951 try:
 952     from graphlib import CycleError, TopologicalSorter
 953 except Exception:
 954     pass
 955 
 956 from hashlib import \
 957     file_digest, md5, pbkdf2_hmac, scrypt, sha1, sha224, sha256, sha384, \
 958     sha512
 959 
 960 from inspect import getfullargspec, getsource
 961 
 962 import itertools
 963 from itertools import \
 964     accumulate, chain, combinations, combinations_with_replacement, \
 965     compress, count, cycle, dropwhile, filterfalse, groupby, islice, \
 966     permutations, product, repeat, starmap, takewhile, tee, zip_longest
 967 try:
 968     from itertools import pairwise
 969     from itertools import batched
 970 except Exception:
 971     pass
 972 
 973 from json import dump, dumps, loads
 974 
 975 import math
 976 Math = math
 977 from math import \
 978     acos, acosh, asin, asinh, atan, atan2, atanh, ceil, comb, \
 979     copysign, cos, cosh, degrees, dist, e, erf, erfc, exp, expm1, \
 980     fabs, factorial, floor, fmod, frexp, fsum, gamma, gcd, hypot, inf, \
 981     isclose, isfinite, isinf, isnan, isqrt, lcm, ldexp, lgamma, log, \
 982     log10, log1p, log2, modf, nan, nextafter, perm, pi, pow, prod, \
 983     radians, remainder, sin, sinh, sqrt, tan, tanh, tau, trunc, ulp
 984 try:
 985     from math import cbrt, exp2
 986 except Exception:
 987     pass
 988 
 989 power = pow
 990 
 991 import operator
 992 
 993 from pathlib import Path
 994 
 995 from pprint import \
 996     isreadable, isrecursive, pformat, pp, pprint, PrettyPrinter, saferepr
 997 
 998 from random import \
 999     betavariate, choice, choices, expovariate, gammavariate, gauss, \
1000     getrandbits, getstate, lognormvariate, normalvariate, paretovariate, \
1001     randbytes, randint, random, randrange, sample, seed, setstate, \
1002     shuffle, triangular, uniform, vonmisesvariate, weibullvariate
1003 
1004 compile_py = compile # keep built-in func compile for later
1005 from re import compile as compile_uncached, Pattern, IGNORECASE
1006 
1007 import statistics
1008 from statistics import \
1009     bisect_left, bisect_right, fmean, \
1010     geometric_mean, harmonic_mean, mean, median, \
1011     median_grouped, median_high, median_low, mode, multimode, pstdev, \
1012     pvariance, quantiles, stdev, variance
1013 try:
1014     from statistics import \
1015         correlation, covariance, linear_regression, mul
1016 except Exception:
1017     pass
1018 
1019 import string
1020 from string import \
1021     Formatter, Template, ascii_letters, ascii_lowercase, ascii_uppercase, \
1022     capwords, digits, hexdigits, octdigits, printable, punctuation, \
1023     whitespace
1024 
1025 alphabet = ascii_letters
1026 letters = ascii_letters
1027 lowercase = ascii_lowercase
1028 uppercase = ascii_uppercase
1029 
1030 from textwrap import dedent, fill, indent, shorten, wrap
1031 
1032 from time import \
1033     altzone, asctime, \
1034     ctime, daylight, get_clock_info, \
1035     gmtime, localtime, mktime, monotonic, monotonic_ns, perf_counter, \
1036     perf_counter_ns, process_time, process_time_ns, \
1037     sleep, strftime, strptime, struct_time, thread_time, thread_time_ns, \
1038     time, time_ns, timezone, tzname
1039 try:
1040     from time import \
1041         clock_getres, clock_gettime, clock_gettime_ns, clock_settime, \
1042         clock_settime_ns, pthread_getcpuclockid, tzset
1043 except Exception:
1044     pass
1045 
1046 from unicodedata import \
1047     bidirectional, category, combining, decimal, decomposition, digit, \
1048     east_asian_width, is_normalized, lookup, mirrored, name, normalize, \
1049     numeric
1050 
1051 from urllib.parse import \
1052     parse_qs, parse_qsl, quote, quote_from_bytes, quote_plus, unquote, \
1053     unquote_plus, unquote_to_bytes, unwrap, urldefrag, urlencode, urljoin, \
1054     urlparse, urlsplit, urlunparse, urlunsplit
1055 
1056 
1057 class Skip:
1058     'Custom type which some funcs type-check to skip values in containers.'
1059 
1060     def __init__(self, *args) -> None:
1061         pass
1062 
1063 
1064 # skip is a ready-to-use value which some funcs filter against: this way
1065 # filtering values becomes a special case of transforming values
1066 skip = Skip()
1067 
1068 # re_cache is used by custom func compile to cache previously-compiled
1069 # regular-expressions, which makes them quicker to (re)use in formulas
1070 re_cache: Dict[str, Pattern] = {}
1071 
1072 # ire_cache is like re_cache, except it's for case-insensitive regexes
1073 ire_cache: Dict[str, Pattern] = {}
1074 
1075 # ansi_style_re detects the most commonly-used ANSI-style sequences, and
1076 # is used in func plain
1077 ansi_style_re = compile_uncached('''\x1b\[([0-9;]+m|[0-9]*[A-HJKST])''')
1078 
1079 # number_re detects numbers, and is used in func numbers
1080 number_re = compile_uncached('''\W(-?[0-9]+(\.[0-9]*)?)\W''')
1081 
1082 # link_re detects web links, and is used in func links
1083 link_re_src = 'https?://[A-Za-z0-9+_.:%-]+(/[A-Za-z0-9+_.%/,#?&=-]*)*'
1084 link_re = compile_uncached(link_re_src)
1085 
1086 # paddable_tab_re detects single tabs and possible runs of spaces around
1087 # them, and is used in func squeeze
1088 paddable_tab_re = compile_uncached(' *\t *')
1089 
1090 # seen remembers values already given to func `once`
1091 seen = set()
1092 
1093 # commented_re detects strings/lines which start as unix-style comments
1094 commented_re = compile_uncached('^ *#')
1095 
1096 # emptyish_re detects empty/emptyish strings/lines, the latter being strings
1097 # with only spaces in them
1098 emptyish_re = compile_uncached('^ *\r?\n?$')
1099 
1100 # spaces_re detects runs of 2 or more spaces, and is used in func squeeze
1101 spaces_re = compile_uncached('  +')
1102 
1103 # awk_sep_re splits like AWK does by default, and is used in func fields
1104 awk_sep_re = compile_uncached(' *\t *| +')
1105 
1106 
1107 # some convenience aliases to commonly-used values
1108 
1109 false = False
1110 true = True
1111 nil = None
1112 nihil = None
1113 none = None
1114 null = None
1115 s = ''
1116 
1117 months = [
1118     'January', 'February', 'March', 'April', 'May', 'June',
1119     'July', 'August', 'September', 'October', 'November', 'December',
1120 ]
1121 
1122 monweek = [
1123     'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
1124     'Saturday', 'Sunday',
1125 ]
1126 
1127 sunweek = [
1128     'Sunday',
1129     'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
1130 ]
1131 
1132 phy = {
1133     'kilo': 1_000,
1134     'mega': 1_000_000,
1135     'giga': 1_000_000_000,
1136     'tera': 1_000_000_000_000,
1137     'peta': 1_000_000_000_000_000,
1138     'exa': 1_000_000_000_000_000_000,
1139     'zetta': 1_000_000_000_000_000_000_000,
1140 
1141     'c': 299_792_458,
1142     'kcd': 683,
1143     'na': 602214076000000000000000,
1144 
1145     'femto': 1e-15,
1146     'pico': 1e-12,
1147     'nano': 1e-9,
1148     'micro': 1e-6,
1149     'milli': 1e-3,
1150 
1151     'e': 1.602176634e-19,
1152     'f': 96_485.33212,
1153     'h': 6.62607015e-34,
1154     'k': 1.380649e-23,
1155     'mu': 1.66053906892e-27,
1156 
1157     'ge': 9.7803267715,
1158     'gn': 9.80665,
1159 }
1160 
1161 physics = phy
1162 
1163 # using literal strings on the cmd-line is often tricky/annoying: some of
1164 # these aliases can help get around multiple levels of string-quoting; no
1165 # quotes are needed as the script will later make these values accessible
1166 # via the property/dot syntax
1167 sym = {
1168     'amp': '&',
1169     'ampersand': '&',
1170     'ansiclear': '\x1b[0m',
1171     'ansinormal': '\x1b[0m',
1172     'ansireset': '\x1b[0m',
1173     'apo': '\'',
1174     'apos': '\'',
1175     'ast': '*',
1176     'asterisk': '*',
1177     'at': '@',
1178     'backquote': '`',
1179     'backslash': '\\',
1180     'backtick': '`',
1181     'ball': '●',
1182     'bang': '!',
1183     'bigsigma': 'Σ',
1184     'block': '█',
1185     'bquo': '`',
1186     'bquote': '`',
1187     'bslash': '\\',
1188     'btick': '`',
1189     'bullet': '•',
1190     'caret': '^',
1191     'cdot': '·',
1192     'circle': '●',
1193     'colon': ':',
1194     'comma': ',',
1195     'cr': '\r',
1196     'crlf': '\r\n',
1197     'cross': '×',
1198     'cs': ', ',
1199     'dash': '—',
1200     'dollar': '$',
1201     'dot': '.',
1202     'dquo': '"',
1203     'dquote': '"',
1204     'emark': '!',
1205     'emdash': '—',
1206     'empty': '',
1207     'endash': '–',
1208     'eq': '=',
1209     'et': '&',
1210     'euro': '€',
1211     'ge': '≥',
1212     'geq': '≥',
1213     'gt': '>',
1214     'hellip': '…',
1215     'hole': '○',
1216     'hyphen': '-',
1217     'infinity': '∞',
1218     'lcurly': '{',
1219     'ldquo': '“',
1220     'ldquote': '“',
1221     'le': '≤',
1222     'leq': '≤',
1223     'lf': '\n',
1224     'lt': '<',
1225     'mdash': '—',
1226     'mdot': '·',
1227     'miniball': '•',
1228     'minus': '-',
1229     'ndash': '–',
1230     'neq': '≠',
1231     'perc': '%',
1232     'percent': '%',
1233     'period': '.',
1234     'plus': '+',
1235     'qmark': '?',
1236     'que': '?',
1237     'rcurly': '}',
1238     'rdquo': '”',
1239     'rdquote': '”',
1240     'sball': '•',
1241     'semi': ';',
1242     'semicolon': ';',
1243     'sharp': '#',
1244     'slash': '/',
1245     'space': ' ',
1246     'square': '■',
1247     'squo': '\'',
1248     'squote': '\'',
1249     'tab': '\t',
1250     'tilde': '~',
1251     'underscore': '_',
1252     'uscore': '_',
1253     'utf8bom': '\xef\xbb\xbf',
1254     'utf16be': '\xfe\xff',
1255     'utf16le': '\xff\xfe',
1256 }
1257 
1258 symbols = sym
1259 
1260 units = {
1261     'cup2l': 0.23658824,
1262     'floz2l': 0.0295735295625,
1263     'floz2ml': 29.5735295625,
1264     'ft2m': 0.3048,
1265     'gal2l': 3.785411784,
1266     'in2cm': 2.54,
1267     'lb2kg': 0.45359237,
1268     'mi2km': 1.609344,
1269     'mpg2kpl': 0.425143707,
1270     'nmi2km': 1.852,
1271     'oz2g': 28.34952312,
1272     'psi2pa': 6894.757293168,
1273     'ton2kg': 907.18474,
1274     'yd2m': 0.9144,
1275 
1276     'mol': 602214076000000000000000,
1277     'mole': 602214076000000000000000,
1278 
1279     'hour': 3_600,
1280     'day': 86_400,
1281     'week': 604_800,
1282 
1283     'hr': 3_600,
1284     'wk': 604_800,
1285 
1286     'kb': 1024,
1287     'mb': 1024**2,
1288     'gb': 1024**3,
1289     'tb': 1024**4,
1290     'pb': 1024**5,
1291 }
1292 
1293 # some convenience aliases to various funcs from the python stdlib
1294 geomean = geometric_mean
1295 harmean = harmonic_mean
1296 sd = stdev
1297 popsd = pstdev
1298 var = variance
1299 popvar = pvariance
1300 randbeta = betavariate
1301 randexp = expovariate
1302 randgamma = gammavariate
1303 randlognorm = lognormvariate
1304 randnorm = normalvariate
1305 randweibull = weibullvariate
1306 
1307 capitalize = str.capitalize
1308 casefold = str.casefold
1309 center = str.center
1310 # count = str.count
1311 decode = bytes.decode
1312 encode = str.encode
1313 endswith = str.endswith
1314 expandtabs = str.expandtabs
1315 find = str.find
1316 format = str.format
1317 index = str.index
1318 isalnum = str.isalnum
1319 isalpha = str.isalpha
1320 isascii = str.isascii
1321 isdecimal = str.isdecimal
1322 isdigit = str.isdigit
1323 isidentifier = str.isidentifier
1324 islower = str.islower
1325 isnumeric = str.isnumeric
1326 isprintable = str.isprintable
1327 isspace = str.isspace
1328 istitle = str.istitle
1329 isupper = str.isupper
1330 # join = str.join
1331 ljust = str.ljust
1332 lower = str.lower
1333 lowered = str.lower
1334 lstrip = str.lstrip
1335 maketrans = str.maketrans
1336 partition = str.partition
1337 removeprefix = str.removeprefix
1338 removesuffix = str.removesuffix
1339 replace = str.replace
1340 rfind = str.rfind
1341 rindex = str.rindex
1342 rjust = str.rjust
1343 rpartition = str.rpartition
1344 rsplit = str.rsplit
1345 rstrip = str.rstrip
1346 # split = str.split
1347 splitlines = str.splitlines
1348 startswith = str.startswith
1349 strip = str.strip
1350 swapcase = str.swapcase
1351 title = str.title
1352 translate = str.translate
1353 upper = str.upper
1354 uppered = str.upper
1355 zfill = str.zfill
1356 
1357 every = all
1358 rev = reversed
1359 reverse = reversed
1360 some = any
1361 
1362 length = len
1363 
1364 blowtabs = str.expandtabs
1365 hasprefix = str.startswith
1366 hassuffix = str.endswith
1367 ltrim = str.lstrip
1368 stripstart = str.lstrip
1369 trimspace = str.strip
1370 trimstart = str.lstrip
1371 rtrim = str.rstrip
1372 stripend = str.rstrip
1373 trimend = str.rstrip
1374 stripped = str.strip
1375 trim = str.strip
1376 trimmed = str.strip
1377 trimprefix = str.removeprefix
1378 trimsuffix = str.removesuffix
1379 
1380 
1381 def required_arg_count(f: Callable) -> int:
1382     if isinstance(f, type):
1383         return 1
1384 
1385     meta = getfullargspec(f)
1386     n = len(meta.args)
1387     if meta.defaults:
1388         n -= len(meta.defaults)
1389     return n
1390 
1391 
1392 def identity(x: Any) -> Any:
1393     '''
1394     Return the value given: this is the default transformer for several
1395     higher-order funcs, which effectively keeps original items as given.
1396     '''
1397     return x
1398 
1399 idem = identity
1400 iden = identity
1401 
1402 
1403 def after(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
1404     'Skip parts of strings/sequences up to the substring/value given.'
1405     return (strafter if isinstance(x, str) else itemsafter)(x, what)
1406 
1407 def afterlast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
1408     'Skip parts of strings/sequences up to the last substring/value given.'
1409     return (strafterlast if isinstance(x, str) else itemsafterlast)(x, what)
1410 
1411 afterfinal = afterlast
1412 
1413 def arrayish(x: Any) -> bool:
1414     'Check if a value is array-like enough.'
1415     return isinstance(x, (list, tuple, range, Generator))
1416 
1417 isarrayish = arrayish
1418 
1419 def base64(x):
1420     return base64bytes(str(x).encode()).decode()
1421 
1422 def basename(s: str) -> str:
1423     'Get a filepath\'s last part, if present.'
1424     return Path(s).name
1425 
1426 def before(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
1427     'End strings/sequences right before a substring/value\'s appearance.'
1428     return (strbefore if isinstance(x, str) else itemsbefore)(x, what)
1429 
1430 def beforelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
1431     'End strings/sequences right before a substring/value\'s last appearance.'
1432     return (strbeforelast if isinstance(x, str) else itemsbeforelast)(x, what)
1433 
1434 beforefinal = beforelast
1435 
1436 def cases(x: Any, *args: Any) -> Any:
1437     '''
1438     Simulate a switch statement on a value, using matches/result pairs from
1439     the arguments given; when given an even number of extra args, None is
1440     used as a final fallback result; when given an odd number of extra args,
1441     the last argument is used as a final `default` value, if needed.
1442     '''
1443 
1444     for i in range(0, len(args) - len(args) % 2, 2):
1445         test, res = args[i], args[i+1]
1446         if isinstance(test, (list, tuple)) and x in test:
1447             return res
1448         if isinstance(test, float) and isnan(test) and isnan(x):
1449             return res
1450         if x == test:
1451             return res
1452     return None if len(args) % 2 == 0 else args[-1]
1453 
1454 switch = cases
1455 
1456 def chunk(items: Iterable, chunk_size: int) -> Iterable:
1457     'Break iterable into chunks, each with up to the item-count given.'
1458 
1459     if isinstance(items, str):
1460         n = len(items)
1461         while n >= chunk_size:
1462             yield items[:chunk_size]
1463             items = items[chunk_size:]
1464             n -= chunk_size
1465         if n > 0:
1466             yield items
1467         return
1468 
1469     if not isinstance(chunk_size, int):
1470         raise Exception('non-integer chunk-size')
1471     if chunk_size < 1:
1472         raise Exception('non-positive chunk-size')
1473 
1474     it = iter(items)
1475     while True:
1476         head = tuple(islice(it, chunk_size))
1477         if not head:
1478             return
1479         yield head
1480 
1481 chunked = chunk
1482 
1483 def commented(s: str) -> bool:
1484     'Check if a string starts as a unix-style comment.'
1485     return commented_re.match(s) != None
1486 
1487 iscommented = commented
1488 
1489 def compile(s: str, case_sensitive: bool = True) -> Pattern:
1490     'Cached regex `compiler`, so it\'s quicker to (re)use in formulas.'
1491 
1492     cache = re_cache if case_sensitive else ire_cache
1493     options = 0 if case_sensitive else IGNORECASE
1494 
1495     if s in cache:
1496         return cache[s]
1497     e = compile_uncached(s, options)
1498     cache[s] = e
1499     return e
1500 
1501 def compose(*what: Callable) -> Callable:
1502     def composite(x: Any) -> Any:
1503         for f in what:
1504             x = f(x)
1505         return x
1506     return composite
1507 
1508 composed = compose
1509 lcompose = compose
1510 lcomposed = compose
1511 
1512 def cond(*args: Any) -> Any:
1513     '''
1514     Simulate a chain of if-else statements, using condition/result pairs
1515     from the arguments given; when given an even number of args, None is
1516     used as a final fallback result; when given an odd number of args, the
1517     last argument is used as a final `else` value, if needed.
1518     '''
1519 
1520     for i in range(0, len(args) - len(args) % 2, 2):
1521         if args[i]:
1522             return args[i+1]
1523     return None if len(args) % 2 == 0 else args[-1]
1524 
1525 def conform(x: Any, denan: Any = None, deinf: Any = None, fn = str) -> Any:
1526     'Make values JSON-compatible.'
1527 
1528     if isinstance(x, float):
1529         # turn NaNs and Infinities into the replacement values given
1530         if isnan(x):
1531             return denan
1532         if isinf(x):
1533             return deinf
1534         return x
1535 
1536     if isinstance(x, (bool, int, float, str)):
1537         return x
1538 
1539     if isinstance(x, dict):
1540         return {
1541             str(k): conform(v) for k, v in x.items() if not
1542                 (isinstance(k, Skip) or isinstance(v, Skip))
1543         }
1544 
1545     if isinstance(x, Iterable):
1546         return [conform(e) for e in x if not isinstance(e, Skip)]
1547 
1548     if isinstance(x, DotCallable):
1549         return x.value
1550 
1551     return fn(x)
1552 
1553 fix = conform
1554 
1555 def countif(src: Iterable, check: Callable) -> int:
1556     '''
1557     Count how many values make the func given true-like. This func works with
1558     sequences, dictionaries, and strings.
1559     '''
1560 
1561     if callable(src):
1562         src, check = check, src
1563     check = predicate(check)
1564 
1565     total = 0
1566     if isinstance(src, dict):
1567         for v in src.values():
1568             if check(v):
1569                 total += 1
1570     else:
1571         for v in src:
1572             if check(v):
1573                 total += 1
1574     return total
1575 
1576 # def debase64(x):
1577 #     return debase64bytes(str(x).encode()).decode()
1578 
1579 def debase64(s: str) -> bytes:
1580     'Convert away from base64 encoding, including data-URIs.'
1581 
1582     if s.startswith('data:'):
1583         i = s.find(',')
1584         if i >= 0:
1585             return standard_b64decode(s[i + 1:])
1586     return standard_b64decode(s)
1587 
1588 unbase64 = debase64
1589 
1590 def dedup(v: Iterable) -> Iterable:
1591     'Ignore reappearing items from iterables, after their first occurrence.'
1592 
1593     got = set()
1594     for e in v:
1595         if not e in got:
1596             got.add(e)
1597             yield e
1598 
1599 dedupe = dedup
1600 deduped = dedup
1601 deduplicate = dedup
1602 deduplicated = dedup
1603 undup = dedup
1604 undupe = dedup
1605 unduped = dedup
1606 unduplicate = dedup
1607 unduplicated = dedup
1608 unique = dedup
1609 uniqued = dedup
1610 
1611 def defunc(x: Any) -> Any:
1612     'Call if value is a func, or return it back as given.'
1613     return x() if callable(x) else x
1614 
1615 callmemaybe = defunc
1616 defunct = defunc
1617 unfunc = defunc
1618 unfunct = defunc
1619 
1620 def dejson(x: Any, catch: Union[Callable[[Exception], Any], Any] = None) -> Any:
1621     'Safely parse JSON from strings.'
1622     try:
1623         return loads(x) if isinstance(x, str) else x
1624     except Exception as e:
1625         return catch(e) if callable(catch) else catch
1626 
1627 unjson = dejson
1628 
1629 def denan(x: Any, fallback: Any = None) -> Any:
1630     'Replace floating-point NaN with the alternative value given.'
1631     return x if not (isinstance(x, float) and isnan(x)) else fallback
1632 
1633 def denil(*args: Any) -> Any:
1634     'Avoid None values, if possible: first value which isn\'t None wins.'
1635     for e in args:
1636         if e != None:
1637             return e
1638     return None
1639 
1640 denone = denil
1641 denull = denil
1642 
1643 def dirname(s: str) -> str:
1644     'Ignore the last part of a filepath.'
1645     return str(Path(s).parent)
1646 
1647 def dive(into: Any, doing: Callable) -> Any:
1648     'Transform a nested value by calling a func via depth-first recursion.'
1649 
1650     # support args in either order
1651     if callable(into):
1652         into, doing = doing, into
1653 
1654     return _dive_kv(None, into, doing)
1655 
1656 deepmap = dive
1657 dive1 = dive
1658 
1659 def divebin(x: Any, y: Any, doing: Callable) -> Any:
1660     'Nested 2-value version of depth-first-recursive func dive.'
1661 
1662     # support args in either order
1663     if callable(x):
1664         x, y, doing = y, doing, x
1665 
1666     narg = required_arg_count(doing)
1667     if narg == 2:
1668         return dive(x, lambda a: dive(y, lambda b: doing(a, b)))
1669     if narg == 4:
1670         return dive(x, lambda i, a: dive(y, lambda j, b: doing(i, a, j, b)))
1671     raise Exception('divebin(...) only supports funcs with 2 or 4 args')
1672 
1673 bindive = divebin
1674 # diveboth = divebin
1675 # dualdive = divebin
1676 # duodive = divebin
1677 dive2 = divebin
1678 
1679 def _dive_kv(key: Any, into: Any, doing: Callable) -> Any:
1680     if isinstance(into, dict):
1681         return {k: _dive_kv(k, v, doing) for k, v in into.items()}
1682     if isinstance(into, Iterable) and not isinstance(into, str):
1683         return [_dive_kv(i, e, doing) for i, e in enumerate(into)]
1684 
1685     narg = required_arg_count(doing)
1686     return doing(key, into) if narg == 2 else doing(into)
1687 
1688 class DotCallable:
1689     'Enable convenient dot-syntax calling of 1-input funcs.'
1690 
1691     def __init__(self, value: Any):
1692         self.value = value
1693 
1694     def __getattr__(self, key: str) -> Any:
1695         return DotCallable(globals()[key](self.value))
1696 
1697 class Dottable:
1698     'Enable convenient dot-syntax access to dictionary values.'
1699 
1700     def __getattr__(self, key: Any) -> Any:
1701         return self.__dict__[key] if key in self.__dict__ else None
1702 
1703     def __getitem__(self, key: Any) -> Any:
1704         return self.__dict__[key] if key in self.__dict__ else None
1705 
1706     def __iter__(self) -> Iterable:
1707         return iter(self.__dict__)
1708 
1709 def dotate(x: Any) -> Union[Dottable, Any]:
1710     'Recursively ensure all dictionaries in a value are dot-accessible.'
1711 
1712     if isinstance(x, dict):
1713         d = Dottable()
1714         d.__dict__ = {k: dotate(v) for k, v in x.items()}
1715         return d
1716     if isinstance(x, list):
1717         return [dotate(e) for e in x]
1718     if isinstance(x, tuple):
1719         return tuple(dotate(e) for e in x)
1720     return x
1721 
1722 dotated = dotate
1723 dote = dotate
1724 doted = dotate
1725 dotified = dotate
1726 dotify = dotate
1727 dottified = dotate
1728 dottify = dotate
1729 
1730 # make dictionaries `physics`, `symbols`, and `units` easier to use
1731 phy = dotate(phy)
1732 physics = phy
1733 sym = dotate(sym)
1734 symbols = sym
1735 units = dotate(units)
1736 
1737 def drop(src: Any, *what) -> Any:
1738     '''
1739     Either ignore all substrings occurrences, or ignore all keys given from
1740     an object, or even from a sequence of objects.
1741     '''
1742 
1743     if isinstance(src, str):
1744         return strdrop(src, *what)
1745     return _itemsdrop(src, set(what))
1746 
1747 dropped = drop
1748 # ignore = drop
1749 # ignored = drop
1750 
1751 def _itemsdrop(src: Any, what: Set) -> Any:
1752     if isinstance(src, dict):
1753         kv = {}
1754         for k, v in src.items():
1755             if not (k in what):
1756                 kv[k] = v
1757         return kv
1758 
1759     if isinstance(src, Iterable):
1760         return [_itemsdrop(e, what) for e in src]
1761 
1762     return None
1763 
1764 def each(src: Iterable, f: Callable) -> Any:
1765     '''
1766     A generalization of built-in func map, which can also handle dictionaries
1767     and strings.
1768     '''
1769 
1770     if callable(src):
1771         src, f = f, src
1772 
1773     if isinstance(src, dict):
1774         return mapkv(src, lambda k, _: k, f)
1775 
1776     if isinstance(src, str):
1777         s = StringIO()
1778         f = loopify(f)
1779         for i, c in enumerate(src):
1780             v = f(i, c)
1781             if not isinstance(v, Skip):
1782                 s.write(str(v))
1783         return s.getvalue()
1784 
1785     return tuple(f(i, v) for i, v in enumerate(src))
1786 
1787 mapped = each
1788 
1789 def emptyish(x: Any) -> bool:
1790     '''
1791     Check if a value can be considered empty, which includes non-empty
1792     strings which only have spaces in them.
1793     '''
1794 
1795     def check(x: Any) -> bool:
1796         if not x:
1797             return True
1798         if isinstance(x, str):
1799             return bool(emptyish_re.match(x))
1800         return False
1801 
1802     if check(x):
1803         return True
1804     if isinstance(x, Iterable):
1805         return all(check(e) for e in x)
1806     return False
1807 
1808 isemptyish = emptyish
1809 
1810 def endict(x: Any) -> Dict[str, Any]:
1811     'Turn non-dictionary values into dictionaries with string keys.'
1812 
1813     if isinstance(x, dict):
1814         return {str(k): v for k, v in x.items()}
1815     if arrayish(x):
1816         return {str(e): e for e in x}
1817     return {str(x): x}
1818 
1819 dicted = endict
1820 endicted = endict
1821 indict = endict
1822 todict = endict
1823 
1824 def enfloat(x: Any, fallback: float = nan) -> float:
1825     try:
1826         return float(x)
1827     except Exception:
1828         return fallback
1829 
1830 enfloated = enfloat
1831 floated = enfloat
1832 floatify = enfloat
1833 floatize = enfloat
1834 tofloat = enfloat
1835 
1836 def enint(x: Any, fallback: Any = None) -> Any:
1837     try:
1838         return int(x)
1839     except Exception:
1840         return fallback
1841 
1842 eninted = enint
1843 inted = enint
1844 integered = enint
1845 intify = enint
1846 intize = enint
1847 toint = enint
1848 
1849 def enlist(x: Any) -> List[Any]:
1850     'Turn non-list values into lists.'
1851     return list(x) if arrayish(x) else [x]
1852 
1853 # inlist = enlist
1854 enlisted = enlist
1855 listify = enlist
1856 listize = enlist
1857 tolist = enlist
1858 
1859 def entuple(x: Any) -> Tuple[Any, ...]:
1860     'Turn non-tuple values into tuples.'
1861     return tuple(x) if arrayish(x) else (x, )
1862 
1863 entupled = entuple
1864 ntuple = entuple
1865 ntupled = entuple
1866 tuplify = entuple
1867 tuplize = entuple
1868 toentuple = entuple
1869 tontuple = entuple
1870 totuple = entuple
1871 
1872 def error(message: Any) -> Exception:
1873     return Exception(str(message))
1874 
1875 err = error
1876 
1877 def ext(s: str) -> str:
1878     'Get a filepath\'s extension, if present.'
1879 
1880     name = Path(s).name
1881     i = name.rfind('.')
1882     return name[i:] if i >= 0 else ''
1883 
1884 filext = ext
1885 
1886 def fail(message: Any, error_code: int = 255) -> NoReturn:
1887     stdout.flush()
1888     print(f'\x1b[31m{message}\x1b[0m', file=stderr)
1889     quit(error_code)
1890 
1891 abort = fail
1892 bail = fail
1893 
1894 def fields(s: str) -> Iterable[str]:
1895     'Split fields AWK-style from the string given.'
1896     return awk_sep_re.split(s.strip())
1897 
1898 # items = fields
1899 splitfields = fields
1900 splititems = fields
1901 words = fields
1902 
1903 def first(items: SupportsIndex, fallback: Any = None) -> Any:
1904     return items[0] if len(items) > 0 else fallback
1905 
1906 def flappend(*args: Any) -> List[Any]:
1907     'Turn arbitrarily-nested values/sequences into a single flat sequence.'
1908 
1909     flat = []
1910     def dig(x: Any) -> None:
1911         if arrayish(x):
1912             for e in x:
1913                 dig(e)
1914         elif isinstance(x, dict):
1915             for e in x.values():
1916                 dig(e)
1917         else:
1918             flat.append(x)
1919 
1920     for e in args:
1921         dig(e)
1922     return flat
1923 
1924 def flat(*args: Any) -> Iterable:
1925     'Turn arbitrarily-nested values/sequences into a single flat sequence.'
1926 
1927     def _flat_rec(x: Any) -> Iterable:
1928         if x is None:
1929             return
1930 
1931         if isinstance(x, dict):
1932             yield from _flat_rec(x.values())
1933 
1934         if isinstance(x, str):
1935             yield x
1936             return
1937 
1938         if isinstance(x, Iterable):
1939             for e in x:
1940                 yield from _flat_rec(e)
1941             return
1942 
1943         yield x
1944 
1945     for x in args:
1946         yield from _flat_rec(x)
1947 
1948 flatten = flat
1949 flattened = flat
1950 
1951 def fromto(start, stop, f: Callable = identity) -> Iterable:
1952     'Sequence all integers between the numbers given, end-value included.'
1953     return (f(e) for e in range(start, stop + 1))
1954 
1955 def fuzz(x: Union[int, float]) -> Union[float, Dict[str, float]]:
1956     '''
1957     Deapproximate numbers to their max range before approximation: the
1958     result is a dictionary with the guessed lower-bound number, the number
1959     given, and the guessed upper-bound number which can approximate to the
1960     original number given. NaNs and the infinities are returned as given,
1961     instead of resulting in a dictionary.
1962     '''
1963 
1964     if isnan(x) or isinf(x):
1965         return x
1966 
1967     if x == 0:
1968         return {'-0.5': -0.5, '0': 0.0, '0.5': +0.5}
1969 
1970     if x % 1 != 0:
1971         # return surrounding integers when given non-integers
1972         a = floor(x)
1973         b = ceil(x)
1974         return {str(a): a, str(x): x, str(b): b}
1975 
1976     if x % 10 != 0:
1977         a = x - 0.5
1978         b = x + 0.5
1979         return {str(a): a, str(x): x, str(b): b}
1980 
1981     # find the integer log10 of the absolute value; 0 was handled previously
1982     y = int(abs(x))
1983     p10 = 1
1984     while True:
1985         if y % p10 != 0:
1986             p10 /= 10
1987             break
1988         p10 *= 10
1989     delta = p10 / 2
1990 
1991     s = +1 if x > 0 else -1
1992     ux = abs(x)
1993     a = s * ux - delta
1994     b = s * ux + delta
1995     return {str(a): a, str(x): x, str(b): b}
1996 
1997 def generated(src: Any) -> Any:
1998     'Make tuples out of generators, or return non-generator values as given.'
1999     return tuple(src) if isinstance(src, (Generator, range)) else src
2000 
2001 concrete = generated
2002 concreted = generated
2003 concretize = generated
2004 concretized = generated
2005 degen = generated
2006 degenerate = generated
2007 degenerated = generated
2008 degenerator = generated
2009 gen = generated
2010 generate = generated
2011 synth = generated
2012 synthed = generated
2013 synthesize = generated
2014 synthesized = generated
2015 
2016 def group(src: Iterable, by: Callable = identity) -> Dict:
2017     '''
2018     Separate transformed items into arrays, the final result being a dict
2019     whose keys are all the transformed values, and whose values are lists
2020     of all the original values which did transform to their group's key.
2021     '''
2022 
2023     if callable(src):
2024         src, by = by, src
2025 
2026     by = loopify(by)
2027     kv = src.items() if isinstance(src, dict) else enumerate(src)
2028 
2029     groups = {}
2030     for k, v in kv:
2031         dk = by(k, v)
2032         if isinstance(dk, Skip) or isinstance(v, Skip):
2033             continue
2034         if dk in groups:
2035             groups[dk].append(v)
2036         else:
2037             groups[dk] = [v]
2038     return groups
2039 
2040 grouped = group
2041 
2042 def gire(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict:
2043     '''
2044     Group matched items into arrays, the final result being a dict whose
2045     keys are all the matchable regexes given, and whose values are lists
2046     of all the original values which did case-insensitively match their
2047     group's key as a regex.
2048     '''
2049 
2050     using = tuple(using)
2051     return group(src, lambda x: imatch(x, using, fallback))
2052 
2053 gbire = gire
2054 groupire = gire
2055 
2056 def gre(src: Iterable[str], using: Iterable[str], fallback: Any = '') -> Dict:
2057     '''
2058     Group matched items into arrays, the final result being a dict whose
2059     keys are all the matchable regexes given, and whose values are lists
2060     of all the original values which did regex-match their group's key.
2061     '''
2062 
2063     using = tuple(using)
2064     return group(src, lambda x: match(x, using, fallback))
2065 
2066 gbre = gre
2067 groupre = gre
2068 
2069 def gsub(s: str, what: str, repl: str) -> str:
2070     'Replace all regex-matches with the string given.'
2071     return compile(what).sub(repl, s)
2072 
2073 def harden(f: Callable, fallback: Any = None) -> Callable:
2074     def _hardened_caller(*args):
2075         try:
2076             return f(*args)
2077         except Exception:
2078             return fallback
2079     return _hardened_caller
2080 
2081 hardened = harden
2082 insure = harden
2083 insured = harden
2084 
2085 def horner(coeffs: List[float], x: Union[int, float]) -> float:
2086     if isinstance(coeffs, (int, float)):
2087         coeffs, x = x, coeffs
2088 
2089     if len(coeffs) == 0:
2090         return 0
2091 
2092     y = coeffs[0]
2093     for c in islice(coeffs, 1, None):
2094         y *= x
2095         y += c
2096     return y
2097 
2098 polyval = horner
2099 
2100 def idiota(n: int, f: Callable = identity) -> Dict[int, int]:
2101     'ID (keys) version of func iota.'
2102     return { v: v for v in (f(e) for e in range(1, n + 1))}
2103 
2104 dictiota = idiota
2105 kviota = idiota
2106 
2107 def imatch(what: str, using: Iterable[str], fallback: str = '') -> str:
2108     'Try to case-insensitively match a string with any of the regexes given.'
2109 
2110     if not isinstance(what, str):
2111         what, using = using, what
2112 
2113     for s in using:
2114         expr = compile(s, False)
2115         m = expr.search(what)
2116         if m:
2117             # return what[m.start():m.end()]
2118             return s
2119     return fallback
2120 
2121 def indices(x: Any) -> Iterable[Any]:
2122     'List all indices/keys, or get an exclusive range from an int.'
2123 
2124     if isinstance(x, int):
2125         return range(x)
2126     if isinstance(x, dict):
2127         return x.keys()
2128     if isinstance(x, (str, list, tuple)):
2129         return range(len(x))
2130     return tuple()
2131 
2132 keys = indices
2133 
2134 def ints(start, stop, f: Callable = identity) -> Iterable[int]:
2135     'Sequence integers, end-value included.'
2136 
2137     if isnan(start) or isnan(stop) or isinf(start) or isinf(stop):
2138         return tuple()
2139     return (f(e) for e in range(int(ceil(start)), int(stop) + 1))
2140 
2141 integers = ints
2142 
2143 def iota(n: int, f: Callable = identity) -> Iterable[int]:
2144     'Sequence all integers from 1 up to (and including) the int given.'
2145     return (f(e) for e in range(1, n + 1))
2146 
2147 def itemsafter(x: Iterable, what: Any) -> Iterable:
2148     ok = False
2149     check = predicate(what)
2150     for e in x:
2151         if ok:
2152             yield e
2153         elif check(e):
2154             ok = True
2155 
2156 def itemsafterlast(x: Iterable, what: Any) -> Iterable:
2157     rest: List[Any] = []
2158     check = predicate(what)
2159     for e in x:
2160         if check(e):
2161             rest.clear()
2162         else:
2163             rest.append(e)
2164 
2165     for e in islice(rest, 1, len(rest)):
2166         yield e
2167 
2168 def itemsbefore(x: Iterable, what: Any) -> Iterable:
2169     check = predicate(what)
2170     for e in x:
2171         if check(e):
2172             return
2173         yield e
2174 
2175 def itemsbeforelast(x: Iterable, what: Any) -> Iterable:
2176     items = []
2177     for e in x:
2178         items.append(e)
2179 
2180     i = -1
2181     check = predicate(what)
2182     for j, e in enumerate(reversed(items)):
2183         if check(e):
2184             i = j
2185             break
2186 
2187     if i < 0:
2188         return items
2189     if i == 0:
2190         return tuple()
2191     for e in islice(items, 0, i):
2192         yield e
2193 
2194 def itemssince(x: Iterable, what: Any) -> Iterable:
2195     ok = False
2196     check = predicate(what)
2197     for e in x:
2198         ok = ok or check(e)
2199         if ok:
2200             yield e
2201 
2202 def itemssincelast(x: Iterable, what: Any) -> Iterable:
2203     rest: List[Any] = []
2204     check = predicate(what)
2205     for e in x:
2206         if check(e):
2207             rest.clear()
2208         else:
2209             rest.append(e)
2210     return rest
2211 
2212 def itemsuntil(x: Iterable, what: Any) -> Iterable:
2213     check = predicate(what)
2214     for e in x:
2215         yield e
2216         if check(e):
2217             return
2218 
2219 def itemsuntillast(x: Iterable, what: Any) -> Iterable:
2220     items = []
2221     for e in x:
2222         items.append(e)
2223 
2224     i = -1
2225     check = predicate(what)
2226     for j, e in enumerate(reversed(items)):
2227         if check(e):
2228             i = j
2229             break
2230 
2231     if i < 0:
2232         return items
2233     for e in islice(items, 0, i + 1):
2234         yield e
2235 
2236 itemsuntilfinal = itemsuntillast
2237 
2238 def join(items: Iterable, sep: Union[str, Iterable] = ' ') -> Union[str, Dict]:
2239     '''
2240     Join iterables using the separator-string given: its 2 arguments
2241     can come in either order, and are sorted out if needed. When given
2242     2 non-string iterables, the result is an object whose keys are from
2243     the first argument, and whose values are from the second one.
2244 
2245     You can use it any of the following ways, where `keys` and `values` are
2246     sequences (lists, tuples, or generators), and `separator` is a string:
2247 
2248         join(values)
2249         join(values, separator)
2250         join(separator, values)
2251         join(keys, values)
2252     '''
2253 
2254     if arrayish(items) and arrayish(sep):
2255         return {k: v for k, v in zip(items, sep)}
2256     if isinstance(items, str):
2257         items, sep = sep, items
2258     return sep.join(str(e) for e in items)
2259 
2260 def joined_paragraphs(lines: Iterable[str]) -> Iterable[Sequence[str]]:
2261     '''
2262     Regroup lines into individual paragraphs, each of which can span multiple
2263     lines: such paragraphs have no empty lines in them, and never end with a
2264     trailing line-feed.
2265     '''
2266 
2267     par: List[str] = []
2268     for l in lines:
2269         if (not l) and par:
2270             yield '\n'.join(par)
2271             par.clear()
2272         else:
2273             par.append(l)
2274 
2275     if len(par) > 0:
2276         yield '\n'.join(par)
2277 
2278 def json0(x: Any) -> str:
2279     'Encode value into a minimal single-line JSON string.'
2280     return dumps(x, separators=(',', ':'), allow_nan=False, indent=None)
2281 
2282 j0 = json0
2283 
2284 def json2(x: Any) -> str:
2285     '''
2286     Encode value into a (possibly multiline) JSON string, using 2 spaces for
2287     each indentation level.
2288     '''
2289     return dumps(x, separators=(',', ': '), allow_nan=False, indent=2)
2290 
2291 j2 = json2
2292 
2293 def jsonl(x: Any) -> Iterable:
2294     'Turn value into multiple JSON-encoded strings, known as JSON Lines.'
2295 
2296     if x is None:
2297         yield dumps(x, allow_nan=False)
2298     elif isinstance(x, (bool, int, float, dict, str)):
2299         yield dumps(x, allow_nan=False)
2300     elif isinstance(x, Iterable):
2301         for e in x:
2302             yield dumps(e, allow_nan=False)
2303     else:
2304         yield dumps(str(x), allow_nan=False)
2305 
2306 jsonlines = jsonl
2307 ndjson = jsonl
2308 tojsonl = jsonl
2309 tojsonlines = jsonl
2310 
2311 def keep(src: Iterable, pred: Any) -> Iterable:
2312     '''
2313     A generalization of built-in func filter, which can also handle dicts and
2314     strings.
2315     '''
2316 
2317     if callable(src):
2318         src, pred = pred, src
2319     pred = predicate(pred)
2320     pred = loopify(pred)
2321 
2322     if isinstance(src, str):
2323         out = StringIO()
2324         for i, c in enumerate(src):
2325             if pred(i, c):
2326                 out.write(c)
2327         return out.getvalue()
2328 
2329     if isinstance(src, dict):
2330         return { k: v for k, v in src.items() if pred(k, v) }
2331     return (e for i, e in enumerate(src) if pred(i, e))
2332 
2333 filtered = keep
2334 kept = keep
2335 
2336 def last(items: SupportsIndex, fallback: Any = None) -> Any:
2337     return items[-1] if len(items) > 0 else fallback
2338 
2339 def links(src: Any) -> Iterable:
2340     'Auto-detect all (HTTP/HTTPS) hyperlink-like substrings.'
2341 
2342     if isinstance(src, str):
2343         for match in link_re.finditer(src):
2344             # yield src[match.start():match.end()]
2345             yield match.group(0)
2346     elif isinstance(src, dict):
2347         for k, v in src.items():
2348             yield from k
2349             yield from links(v)
2350     elif isinstance(src, Iterable):
2351         for v in src:
2352             yield from links(v)
2353 
2354 def loopify(x: Callable) -> Callable:
2355     nargs = required_arg_count(x)
2356     if nargs == 2:
2357         return x
2358     elif nargs == 1:
2359         return lambda _, v: x(v)
2360     else:
2361         raise Exception('only funcs with 1 or 2 args are supported')
2362 
2363 def mapkv(src: Iterable, key: Callable, value: Callable = identity) -> Dict:
2364     '''
2365     A map-like func for dictionaries, which uses 2 mapping funcs, the first
2366     for the keys, the second for the values.
2367     '''
2368 
2369     if key is None:
2370         key = lambda k, _: k
2371 
2372     if callable(src):
2373         src, key, value = value, src, key
2374 
2375     if required_arg_count(key) != 2:
2376         oldkey = key
2377         key = lambda k, _: oldkey(k)
2378 
2379     key = loopify(key)
2380     value = loopify(value)
2381     # if isinstance(src, dict):
2382     #     return { key(k, v): value(k, v) for k, v in src.items() }
2383     # return { key(i, v): value(i, v) for i, v in enumerate(src) }
2384 
2385     def add(k, v, to):
2386         dk = key(k, v)
2387         dv = value(k, v)
2388         if isinstance(dk, Skip) or isinstance(dv, Skip):
2389             return
2390         to[dk] = dv
2391 
2392     res = {}
2393     kv = src.items() if isinstance(src, dict) else enumerate(src)
2394     for k, v in kv:
2395         add(k, v, res)
2396     return res
2397 
2398 def match(what: str, using: Iterable[str], fallback: str = '') -> str:
2399     'Try to match a string with any of the regexes given.'
2400 
2401     if not isinstance(what, str):
2402         what, using = using, what
2403 
2404     for s in using:
2405         expr = compile(s)
2406         m = expr.search(what)
2407         if m:
2408             # return what[m.start():m.end()]
2409             return s
2410     return fallback
2411 
2412 def maybe(f: Callable, x: Any) -> Any:
2413     '''
2414     Try calling a func on a value, using the same value as a fallback result,
2415     in case of exceptions.
2416     '''
2417 
2418     if not callable(f):
2419         f, x = x, f
2420     try:
2421         return f(x)
2422     except Exception:
2423         return x
2424 
2425 def mappend(*args) -> Dict:
2426     kv = {}
2427     for src in args:
2428         if isinstance(src, dict):
2429             for k, v in src.items():
2430                 kv[k] = v
2431         else:
2432             raise Exception('mappend only works with dictionaries')
2433     return kv
2434 
2435 def message(x: Any, result: Any = skip) -> Any:
2436     print(x, file=stderr)
2437     return result
2438 
2439 msg = message
2440 
2441 def must(cond: Any, errmsg: str = 'condition given not always true') -> None:
2442     'Enforce conditions, raising an exception on failure.'
2443     if not cond:
2444         raise Exception(errmsg)
2445 
2446 demand = must
2447 enforce = must
2448 
2449 def nowdict() -> dict:
2450     v = datetime(2000, 1, 1).now()
2451     return {
2452         'year': v.year,
2453         'month': v.month,
2454         'day': v.day,
2455         'hour': v.hour,
2456         'minute': v.minute,
2457         'second': v.second,
2458         'text': v.strftime('%Y-%m-%d %H:%M:%S %b %a'),
2459         'weekday': v.strftime('%A'),
2460     }
2461 
2462 def number(x: Any) -> Union[int, float, Any]:
2463     '''
2464     Try to turn the value given into a number, using a fallback value instead
2465     of raising exceptions.
2466     '''
2467 
2468     if isinstance(x, float):
2469         return x
2470 
2471     try:
2472         return int(x)
2473     except Exception:
2474         return float(x)
2475 
2476 def numbers(src: Any) -> Iterable:
2477     'Auto-detect all number-like substrings.'
2478 
2479     if isinstance(src, str):
2480         for match in number_re.finditer(src):
2481             yield match.group(0).strip()
2482             # yield src[match.start():match.end()].strip()
2483     elif isinstance(src, dict):
2484         for k, v in src.items():
2485             yield from k
2486             yield from links(v)
2487     elif isinstance(src, Iterable):
2488         for v in src:
2489             yield from links(v)
2490 
2491 def numsign(x: Union[int, float]) -> Union[int, float]:
2492     'Get a number\'s sign, or NaN if the number given is a NaN.'
2493 
2494     if isinstance(x, int):
2495         if x > 0:
2496             return +1
2497         if x < 0:
2498             return -1
2499         return 0
2500 
2501     if isnan(x):
2502         return x
2503 
2504     if x > 0:
2505         return +1.0
2506     if x < 0:
2507         return -1.0
2508     return 0.0
2509 
2510 def numstats(src: Any) -> Dict[str, Union[float, int]]:
2511     'Gather several single-pass numeric statistics.'
2512 
2513     n = mean_sq = ln_sum = 0
2514     least = +inf
2515     most = -inf
2516     total = mean = 0
2517     prod = 1
2518     nans = ints = pos = zero = neg = 0
2519 
2520     def update_numstats(x: Any) -> None:
2521         nonlocal nans, n, ints, pos, neg, zero, least, most, total, prod
2522         nonlocal ln_sum, mean, mean_sq
2523 
2524         if not isinstance(x, (float, int)):
2525             return
2526 
2527         if isnan(x):
2528             nans += 1
2529             return
2530 
2531         n += 1
2532         ints += int(isinstance(x, int) or x == floor(x))
2533 
2534         if x > 0:
2535             pos += 1
2536         elif x < 0:
2537             neg += 1
2538         else:
2539             zero += 1
2540 
2541         least = min(least, x)
2542         most = max(most, x)
2543 
2544         # total += x
2545         prod *= x
2546         ln_sum += log(x)
2547 
2548         d1 = x - mean
2549         mean += d1 / n
2550         d2 = x - mean
2551         mean_sq += d1 * d2
2552 
2553     def _numstats_rec(src: Any) -> None:
2554         if isinstance(src, dict):
2555             for e in src.values():
2556                 _numstats_rec(e)
2557         elif isinstance(src, Iterable) and not isinstance(src, str):
2558             for e in src:
2559                 _numstats_rec(e)
2560         else:
2561             update_numstats(src)
2562 
2563     _numstats_rec(src)
2564 
2565     sd = nan
2566     geomean = nan
2567     if n > 0:
2568         sd = sqrt(mean_sq / n)
2569         geomean = exp(ln_sum / n) if not isinf(ln_sum) else nan
2570     total = n * mean
2571 
2572     return {
2573         'n': n,
2574         'nan': nans,
2575         'min': least,
2576         'max': most,
2577         'sum': total,
2578         'mean': mean,
2579         'geomean': geomean,
2580         'sd': sd,
2581         'product': prod,
2582         'integer': ints,
2583         'positive': pos,
2584         'zero': zero,
2585         'negative': neg,
2586     }
2587 
2588 def once(x: Any, replacement: Any = None) -> Any:
2589     '''
2590     Replace the first argument given after the first time this func has been
2591     given it: this is a deliberately stateful function, given its purpose.
2592     '''
2593 
2594     if not (x in seen):
2595         seen.add(x)
2596         return x
2597     else:
2598         return replacement
2599 
2600 onced = once
2601 
2602 def pad(s: str, n: int, pad: str = ' ') -> str:
2603     l = len(s)
2604     return s if l >= n else s + int((n - l) / len(pad)) * pad
2605 
2606 def padcenter(s: str, n: int, pad: str = ' ') -> str:
2607     return s.center(n, pad)
2608 
2609 centerpad = padcenter
2610 centerpadded = padcenter
2611 cjust = padcenter
2612 cpad = padcenter
2613 padc = padcenter
2614 paddedcenter = padcenter
2615 
2616 def padend(s: str, n: int, pad: str = ' ') -> str:
2617     return s.rjust(n, pad)
2618 
2619 padr = padend
2620 padright = padend
2621 paddedend = padend
2622 paddedright = padend
2623 rpad = padend
2624 rightpad = padend
2625 rightpadded = padend
2626 
2627 def padstart(s: str, n: int, pad: str = ' ') -> str:
2628     return s.ljust(n, pad)
2629 
2630 lpad = padstart
2631 leftpad = padstart
2632 leftpadded = padstart
2633 padl = padstart
2634 padleft = padstart
2635 paddedleft = padstart
2636 paddedstart = padstart
2637 
2638 def panic(x: Any) -> None:
2639     raise Exception(x)
2640 
2641 def paragraphize(lines: Iterable[str]) -> Iterable[Sequence[str]]:
2642     '''
2643     Regroup lines into individual paragraphs, each of which is a list of
2644     single-line strings, none of which never end with a trailing line-feed.
2645     '''
2646 
2647     par: List[str] = []
2648     for l in lines:
2649         if (not l) and par:
2650             yield par
2651             par.clear()
2652         else:
2653             par.append(l)
2654 
2655     if len(par) > 0:
2656         yield par
2657 
2658 paragraphed = paragraphize
2659 paragraphs = paragraphize
2660 paragroup = paragraphize
2661 pargroup = paragraphize
2662 
2663 def parse(s: str, fallback: Any = None) -> Any:
2664     'Try to parse JSON, ignoring exceptions in favor of a fallback value.'
2665 
2666     try:
2667         return loads(s)
2668     except Exception:
2669         return fallback
2670 
2671 fromjson = parse
2672 parsed = parse
2673 loaded = parse
2674 unjson = parse
2675 
2676 def pick(src: Any, *what) -> Any:
2677     'Pick only the keys given from an object, or even a sequence of objects.'
2678 
2679     if isinstance(src, dict):
2680         kv = {}
2681         for k in what:
2682             kv[k] = src[k]
2683         return kv
2684 
2685     if isinstance(src, Iterable):
2686         return [pick(e, *what) for e in src]
2687 
2688     return None
2689 
2690 picked = pick
2691 
2692 def plain(s: str) -> str:
2693     'Ignore all ANSI-style sequences in a string.'
2694     return ansi_style_re.sub('', s)
2695 
2696 def predicate(x: Any) -> Callable:
2697     'Helps various higher-order funcs, by standardizing `predicate` values.'
2698 
2699     if callable(x):
2700         return x
2701 
2702     if isinstance(x, float):
2703         if isnan(x):
2704             return lambda y: isinstance(y, float) and isnan(y)
2705         if isinf(x):
2706             return lambda y: isinstance(y, float) and isinf(y)
2707 
2708     return lambda y: x == y
2709 
2710 pred = predicate
2711 
2712 def quoted(s: str, quote: str = '"') -> str:
2713     'Surround a string with quotes.'
2714     return f'{quote}{s}{quote}'
2715 
2716 def recover(*args) -> Any:
2717     '''
2718     Catch exceptions using a lambda/callback func, in one of 6 ways
2719         recover(zero_args_func)
2720         recover(zero_args_func, exception_replacement_value)
2721         recover(zero_args_func, one_arg_exception_handling_func)
2722         recover(one_arg_func, arg)
2723         recover(one_arg_func, arg, exception_replacement_value)
2724         recover(one_arg_func, arg, one_arg_exception_handling_func)
2725     '''
2726 
2727     if len(args) == 1:
2728         f = args[0]
2729         try:
2730             return f()
2731         except Exception:
2732             return None
2733     elif len(args) == 2:
2734         f, fallback = args[0], args[1]
2735         if callable(f) and callable(fallback):
2736             try:
2737                 return f()
2738             except Exception as e:
2739                 nargs = required_arg_count(fallback)
2740                 return fallback(e) if nargs == 1 else fallback()
2741         else:
2742             try:
2743                 return f() if required_arg_count(f) == 0 else f(args[1])
2744             except Exception:
2745                 return fallback
2746     elif len(args) == 3:
2747         f, x, fallback = args[0], args[1], args[2]
2748         if callable(f) and callable(fallback):
2749             try:
2750                 return f(x)
2751             except Exception as e:
2752                 nargs = required_arg_count(fallback)
2753                 return fallback(e) if nargs == 1 else fallback()
2754         else:
2755             try:
2756                 return f(x)
2757             except Exception:
2758                 return fallback
2759     else:
2760         raise Exception('recover(...) only works with 1, 2, or 3 args')
2761 
2762 attempt = recover
2763 attempted = recover
2764 recovered = recover
2765 recoverred = recover
2766 rescue = recover
2767 rescued = recover
2768 trycall = recover
2769 
2770 def reject(src: Iterable, pred: Any) -> Iterable:
2771     '''
2772     A generalization of built-in func filter, which uses predicate funcs the
2773     opposite way, and which can also handle dicts and strings.
2774     '''
2775 
2776     if callable(src):
2777         src, pred = pred, src
2778     pred = predicate(pred)
2779     pred = loopify(pred)
2780 
2781     if isinstance(src, str):
2782         out = StringIO()
2783         for i, c in enumerate(src):
2784             if not pred(i, c):
2785                 out.write(c)
2786         return out.getvalue()
2787 
2788     if isinstance(src, dict):
2789         return { k: v for k, v in src.items() if not pred(k, v) }
2790     return (e for i, e in enumerate(src) if not pred(i, e))
2791 
2792 avoid = reject
2793 avoided = reject
2794 keepout = reject
2795 keptout = reject
2796 rejected = reject
2797 
2798 def retype(x: Any) -> Any:
2799     'Try to narrow the type of the value given.'
2800 
2801     if isinstance(x, float):
2802         return int(x) if floor(x) == x else x
2803 
2804     if not isinstance(x, str):
2805         return x
2806 
2807     try:
2808         return loads(x)
2809     except Exception:
2810         pass
2811 
2812     try:
2813         return int(x)
2814     except Exception:
2815         pass
2816 
2817     try:
2818         return float(x)
2819     except Exception:
2820         pass
2821 
2822     return x
2823 
2824 autocast = retype
2825 mold = retype
2826 molded = retype
2827 narrow = retype
2828 narrowed = retype
2829 recast = retype
2830 recasted = retype
2831 remold = retype
2832 remolded = retype
2833 retyped = retype
2834 
2835 def revcompose(*what: Callable) -> Callable:
2836     def composite(x: Any) -> Any:
2837         for f in reversed(what):
2838             x = f(x)
2839         return x
2840     return composite
2841 
2842 rcompose = revcompose
2843 rcomposed = revcompose
2844 revcomposed = revcompose
2845 
2846 def revsort(iterable: Iterable, key: Optional[Callable] = None) -> Iterable:
2847     return sorted(iterable, key=key, reverse=True)
2848 
2849 revsorted = revsort
2850 
2851 # def revsortkv(src: Dict, key: Callable = None) -> Dict:
2852 #     if not key:
2853 #         key = lambda kv: (kv[1], kv[0])
2854 #     return sortkv(src, key, reverse=True)
2855 
2856 def revsortkv(src: Dict, key: Callable = None) -> Dict:
2857     if key is None:
2858         key = lambda x: x[1]
2859     return sortkv(src, key, reverse=True)
2860 
2861 revsortedkv = revsortkv
2862 
2863 def rstripdecs(s: str) -> str:
2864     '''
2865     Ignore trailing zero decimals on number-like strings; even ignore
2866     the decimal dot if trailing.
2867     '''
2868 
2869     try:
2870         f = float(s)
2871         if isnan(f) or isinf(f):
2872             return s
2873 
2874         dot = s.find('.')
2875         if dot < 0:
2876             return s
2877 
2878         s = s.rstrip('0')
2879         return s[:-1] if s.endswith('.') else s
2880     except Exception:
2881         return s
2882 
2883 chopdecs = rstripdecs
2884 
2885 def scale(x: float, x0: float, x1: float, y0: float, y1: float) -> float:
2886     'Transform a value from a linear domain into another linear one.'
2887     return (y1 - y0) * (x - x0) / (x1 - x0) + y0
2888 
2889 rescale = scale
2890 rescaled = scale
2891 scaled = scale
2892 
2893 def shortened(s: str, maxlen: int, trailer: str = '') -> str:
2894     'Limit strings to the symbol-count given, including an optional trailer.'
2895     maxlen = max(maxlen, 0)
2896     return s if len(s) <= maxlen else s[:maxlen - len(trailer)] + trailer
2897 
2898 def shuffled(x: Any) -> Any:
2899     'Return a shuffled copy of the list given.'
2900     y = copy(x)
2901     shuffle(y)
2902     return y
2903 
2904 def split(src: Union[str, Sequence], n: Union[str, int]) -> Iterable:
2905     'Split/break a string/sequence into several chunks/parts.'
2906 
2907     if isinstance(src, str) and isinstance(n, str):
2908         return src.split(n)
2909     if not (isinstance(src, (str, Sequence)) and isinstance(n, int)):
2910         raise Exception('unsupported type-pair of arguments')
2911 
2912     if n < 1:
2913         return []
2914 
2915     l = len(src)
2916     if l <= n:
2917         return src.split('') if isinstance(src, str) else src
2918 
2919     chunks = []
2920     csize = int(ceil(l / n))
2921     while len(src) > 0:
2922         chunks.append(src[:csize])
2923         src = src[csize:]
2924     return chunks
2925 
2926 broken = split
2927 splitted = split
2928 splitten = split
2929 
2930 def strdrop(x: str, *what: str) -> str:
2931     'Ignore all occurrences of all substrings given.'
2932 
2933     for s in what:
2934         x = x.replace(s, '')
2935     return x
2936 
2937 strignore = strdrop
2938 
2939 def stringify(x: Any) -> str:
2940     'Fancy alias for func dumps, named after JavaScript\'s func.'
2941     return dumps(x, separators=(', ', ': '), allow_nan=False, indent=None)
2942 
2943 jsonate = stringify
2944 jsonify = stringify
2945 tojson = stringify
2946 
2947 def strafter(x: str, what: str) -> str:
2948     i = x.find(what)
2949     return '' if i < 0 else x[i+len(what):]
2950 
2951 def strafterlast(x: str, what: str) -> str:
2952     i = x.rfind(what)
2953     return '' if i < 0 else x[i+len(what):]
2954 
2955 def strbefore(x: str, what: str) -> str:
2956     i = x.find(what)
2957     return x if i < 0 else x[:i]
2958 
2959 def strbeforelast(x: str, what: str) -> str:
2960     i = x.rfind(what)
2961     return x if i < 0 else x[:i]
2962 
2963 def strsince(x: str, what: str) -> str:
2964     i = x.find(what)
2965     return '' if i < 0 else x[i:]
2966 
2967 def strsincelast(x: str, what: str) -> str:
2968     i = x.rfind(what)
2969     return '' if i < 0 else x[i:]
2970 
2971 def struntil(x: str, what: str) -> str:
2972     i = x.find(what)
2973     return x if i < 0 else x[:i+len(what)]
2974 
2975 def struntillast(x: str, what: str) -> str:
2976     i = x.rfind(what)
2977     return x if i < 0 else x[:i+len(what)]
2978 
2979 struntilfinal = struntillast
2980 
2981 def since(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
2982     'Start strings/sequences with a substring/value\'s appearance.'
2983     return (strsince if isinstance(x, str) else itemssince)(x, what)
2984 
2985 def sincelast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
2986     'Start strings/sequences with a substring/value\'s last appearance.'
2987     return (strsincelast if isinstance(x, str) else itemssincelast)(x, what)
2988 
2989 sincefinal = sincelast
2990 
2991 def sortk(x: Dict, key: Callable = identity, reverse: bool = False) -> Dict:
2992     keys = sorted(x.keys(), key=key, reverse=reverse)
2993     return {k: x[k] for k in keys}
2994 
2995 sortkeys = sortk
2996 sortedkeys = sortk
2997 
2998 def sortkv(src: Dict, key: Callable = None, reverse: bool = False) -> Dict:
2999     if key is None:
3000         key = lambda x: x[1]
3001     kv = sorted(src.items(), key=key, reverse=reverse)
3002     return {k: v for (k, v) in kv}
3003 
3004 sortedkv = sortkv
3005 
3006 def squeeze(s: str) -> str:
3007     '''
3008     A more aggressive way to rid strings of extra spaces which, unlike string
3009     method strip, also turns inner runs of multiple spaces into single ones.
3010     '''
3011     s = s.strip()
3012     s = spaces_re.sub(' ', s)
3013     s = paddable_tab_re.sub('\t', s)
3014     return s
3015 
3016 squeezed = squeeze
3017 
3018 def stround(x: Union[int, float], decimals: int = 6) -> str:
3019     'Format numbers into a string with the given decimal-digit count.'
3020 
3021     if decimals >= 0:
3022         return f'{x:.{decimals}f}'
3023     else:
3024         return f'{round(x, decimals):.0f}'
3025 
3026 def tally(src: Iterable, by: Callable = identity) -> Dict[Any, int]:
3027     '''
3028     Count all distinct (transformed) values, the result being a dictionary
3029     whose keys are all the transformed values, and whose items are positive
3030     integers.
3031     '''
3032 
3033     if callable(src):
3034         src, by = by, src
3035 
3036     tally: Dict[Any, int] = {}
3037     by = loopify(by)
3038 
3039     if isinstance(src, dict):
3040         for k, v in src.items():
3041             dk = by(k, v)
3042             if dk in tally:
3043                 tally[dk] += 1
3044             else:
3045                 tally[dk] = 1
3046     else:
3047         for i, v in enumerate(src):
3048             dk = by(i, v)
3049             if dk in tally:
3050                 tally[dk] += 1
3051             else:
3052                 tally[dk] = 1
3053     return tally
3054 
3055 tallied = tally
3056 
3057 def transpose(src: Any) -> Any:
3058     'Turn lists/objects inside-out like socks, so to speak.'
3059 
3060     if isinstance(src, dict):
3061         return { v: k for k, v in src.items() }
3062 
3063     if not arrayish(src):
3064         msg = 'transpose only supports objects or iterables of objects'
3065         raise ValueError(msg)
3066 
3067     kv: Dict[Any, Any] = {}
3068     seq: List[Any] = []
3069 
3070     for e in src:
3071         if isinstance(e, dict):
3072             for k, v in e.items():
3073                 if k in kv:
3074                     kv[k].append(v)
3075                 else:
3076                     kv[k] = [v]
3077         elif isinstance(e, Iterable):
3078             for i, v in enumerate(e):
3079                 if i < len(seq):
3080                     seq[i].append(v)
3081                 else:
3082                     seq.append([v])
3083         else:
3084             msg = 'transpose(...): not all items are iterables/objects'
3085             raise ValueError(msg)
3086 
3087     if len(kv) > 0 and len(seq) > 0:
3088         msg = 'transpose(...): mix of iterables and objects not supported'
3089         raise ValueError(msg)
3090     return kv if len(seq) == 0 else seq
3091 
3092 tr = transpose
3093 transp = transpose
3094 transposed = transpose
3095 
3096 def trap(x: Callable, y: Union[Callable[[Exception], Any], Any] = None) -> Any:
3097     'Try running a func, handing exceptions over to a fallback func.'
3098 
3099     try:
3100         return x() if callable(x) else x
3101     except Exception as e:
3102         if callable(y):
3103             nargs = required_arg_count(y)
3104             return y(e) if nargs == 1 else y()
3105         else:
3106             return y
3107 
3108 catch = trap
3109 catched = trap
3110 caught = trap
3111 noerr = trap
3112 noerror = trap
3113 noerrors = trap
3114 safe = trap
3115 save = trap
3116 saved = trap
3117 trapped = trap
3118 
3119 def tsv(x: str, fn: Union[Callable, None] = None) -> Any:
3120     if fn is None:
3121         return x.split('\t')
3122     if callable(x):
3123         x, fn = fn, x
3124     return fn(x.split('\t'))
3125 
3126 def typename(x: Any) -> str:
3127     if x is None:
3128         return 'null'
3129     if isinstance(x, bool):
3130         return 'boolean'
3131     if isinstance(x, str):
3132         return 'string'
3133     if isinstance(x, (int, float)):
3134         return 'number'
3135     if isinstance(x, (list, tuple)):
3136         return 'array'
3137     if isinstance(x, dict):
3138         return 'object'
3139     return type(x).__name__
3140 
3141 def typeof(x: Any) -> str:
3142     'Get a value\'s JS-like typeof type-string.'
3143 
3144     if callable(x):
3145         return 'function'
3146 
3147     return {
3148         bool: 'boolean',
3149         int: 'number',
3150         float: 'number',
3151         str: 'string',
3152     }.get(type(x), 'object')
3153 
3154 def unixify(s: str) -> str:
3155     '''
3156     Make plain-text `unix-style`, ignoring a leading UTF-8 BOM if present,
3157     and turning any/all CRLF byte-pairs into line-feed bytes.
3158     '''
3159     s = s.lstrip('\xef\xbb\xbf')
3160     return s.replace('\r\n', '\n') if '\r\n' in s else s
3161 
3162 def unquoted(s: str) -> str:
3163     'Ignore surrounding quotes in a string.'
3164 
3165     if s.startswith('"') and s.endswith('"'):
3166         return s[1:-1]
3167     if s.startswith('\'') and s.endswith('\''):
3168         return s[1:-1]
3169     if s.startswith('`') and s.endswith('`'):
3170         return s[1:-1]
3171     if s.startswith('”') and s.endswith('“'):
3172         return s[1:-1]
3173     if s.startswith('“') and s.endswith('”'):
3174         return s[1:-1]
3175     return s
3176 
3177 dequote = unquoted
3178 dequoted = unquoted
3179 
3180 def until(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
3181     'End strings/sequences with a substring/value\'s appearance.'
3182     return (struntil if isinstance(x, str) else itemsuntil)(x, what)
3183 
3184 def untillast(x: Union[str, Iterable], what: Any) -> Union[str, Iterable]:
3185     'End strings/sequences with a substring/value\'s last appearance.'
3186     return (struntillast if isinstance(x, str) else itemsuntillast)(x, what)
3187 
3188 untilfinal = untillast
3189 
3190 
3191 def wait(seconds: Union[int, float], result: Any) -> Any:
3192     'Wait the given number of seconds, before returning its latter arg.'
3193 
3194     t = (int, float)
3195     if (not isinstance(seconds, t)) and isinstance(result, t):
3196         seconds, result = result, seconds
3197     sleep(seconds)
3198     return result
3199 
3200 delay = wait
3201 
3202 def wat(*args) -> None:
3203     'What Are These (wat) shows help/doc messages for funcs given to it.'
3204 
3205     from pydoc import doc
3206 
3207     c = 0
3208     w = stderr
3209 
3210     for e in args:
3211         if not callable(e):
3212             continue
3213 
3214         if c > 0:
3215             print(file=w)
3216 
3217         print(f'\x1b[48;5;253m\x1b[38;5;26m{e.__name__:80}\x1b[0m', file=w)
3218         doc(e, output=w)
3219         c += 1
3220 
3221     return Skip()
3222 
3223 def wit(*args) -> None:
3224     'What Is This (wit) shows help/doc messages for funcs given to it.'
3225     return wat(*args)
3226 
3227 def zoom(x: Any, *keys_indices) -> Any:
3228     for k in keys_indices:
3229         # allow int-indexing dicts the same way lists/tuples can be
3230         if isinstance(x, dict) and isinstance(k, int):
3231             l = len(x)
3232             if i < 0:
3233                 i += l
3234             if i < 0 or i >= len(x):
3235                 x = None
3236                 continue
3237             for i, e in enumerate(x.values()):
3238                 if i == k:
3239                     x = e
3240                     break
3241             continue
3242 
3243         # regular key/index access for dicts/lists/tuples
3244         x = x[k]
3245 
3246     return x
3247 
3248 
3249 # args is the `proper` list of arguments given to the script
3250 args = argv[1:]
3251 run_mode = ''
3252 trace_exceptions = False
3253 profile_run = False
3254 pipe_mode = False
3255 
3256 if len(args) == 0:
3257     # show help message when given no arguments
3258     print(info.strip(), file=stderr)
3259     exit(0)
3260 
3261 trace_opts = (
3262     '-t', '--t', '-trace', '--trace', '-traceback', '--traceback',
3263 )
3264 pipe_opts = ('-p', '--p', '-pipe', '--pipe')
3265 profile_opts = ('-p', '--p', '-prof', '--prof', '-profile', '--profile')
3266 
3267 # handle all other leading options; the explicit help options are
3268 # handled earlier in the script
3269 while len(args) > 0:
3270     if args[0] == '--':
3271         args = args[1:]
3272         break
3273 
3274     if args[0] in trace_opts:
3275         trace_exceptions = True
3276         args = args[1:]
3277         continue
3278 
3279     if args[0] in pipe_opts:
3280         pipe_mode = True
3281         args = args[1:]
3282         break
3283 
3284     if args[0] in profile_opts:
3285         profile_run = True
3286         args = args[1:]
3287         continue
3288 
3289     s = opts2modes.get(args[0], '')
3290     if not s:
3291         break
3292 
3293     run_mode = s
3294     args = args[1:]
3295 
3296 inputs = []
3297 expression = ''
3298 if len(args) > 0:
3299     expression = args[0]
3300     inputs = args[1:]
3301 
3302 if not run_mode:
3303     run_mode = 'each-line'
3304 
3305 if not expression and not (run_mode in ('json-lines', 'each-line')):
3306     # show help message when given no expression
3307     print(info.strip(), file=stderr)
3308     exit(0)
3309 
3310 glo = globals()
3311 for e in (physics, symbols, units):
3312     for k, v in e.__dict__.items():
3313         if not k in glo:
3314             glo[k] = v
3315 
3316 exec = disabled_exec
3317 
3318 
3319 def handle_pipe(src, funcs):
3320     # `comprehension` expressions seem to ignore local variables: even
3321     # lambda-based workarounds fail
3322     global i, n, l, line, v, val, value, e, err, error
3323     # variable names `o` and `p` work like in the `pyp` tool, except
3324     # the pipeline steps were given as separate cmd-line arguments
3325     global o, p
3326 
3327     i = 0
3328     n = 1
3329     e = err = error = None
3330 
3331     for l in src:
3332         l = l.rstrip('\r\n').rstrip('\n')
3333         if i == 0:
3334             l = l.lstrip('\xef\xbb\xbf')
3335 
3336         line = l
3337         o = p = prev = line
3338         # seen is used by func `once` to remember previously-given values
3339         seen.clear()
3340 
3341         try:
3342             e = err = error = None
3343             v = val = value = loads(l)
3344         except BrokenPipeError as e:
3345             raise e
3346         except Exception as ex:
3347             e = err = error = ex
3348             v = val = value = Skip()
3349 
3350         for f in funcs:
3351             p = f(p)
3352             if callable(p):
3353                 p = p(prev)
3354             prev = p
3355 
3356         res = p
3357         i += 1
3358         n += 1
3359 
3360         if isinstance(res, (list, range, tuple, Generator)):
3361             for e in res:
3362                 e = adapt_result(e, None)
3363                 if not (e is None):
3364                     print(e, flush=True)
3365             continue
3366 
3367         res = adapt_result(res, line)
3368         if not (res is None):
3369             print(res, flush=True)
3370 
3371 
3372 try:
3373     if pipe_mode:
3374         steps = [eval(s) for s in args]
3375         compile = None
3376         eval = None
3377         exec = None
3378         open = None
3379         handle_pipe(stdin, steps)
3380         exit(0)
3381 
3382     # compile the expression to speed it up, since they're all (re)run
3383     # for each line from standard input; also, handle a single-dot as
3384     # an identity expression, using the current line as is
3385     if expression in ('', '.'):
3386         expression = {
3387             'all-lines': 'lines',
3388             'all-bytes': 'data',
3389             'each-block': 'block',
3390             'each-line': 'line',
3391             'json-lines': 'data',
3392             'no-input': 'info.strip()',
3393             'whole-strings': 'value',
3394         }[run_mode]
3395     expression = compile_py(expression, expression, 'eval')
3396 
3397     # `comprehension` expressions seem to ignore local variables: even
3398     # lambda-based workarounds fail
3399     i = 0
3400     c = 1
3401     nr = 1
3402     _ = None
3403 
3404     fn = {
3405         'each-line': stop_normal,
3406         'each-block': stop_normal,
3407         'all-lines': stop_normal,
3408         'all-bytes': stop_normal,
3409         'json-lines': stop_json,
3410         'no-input': stop_normal,
3411         'whole-strings': stop_normal,
3412     }[run_mode]
3413     glo['halt'] = fn
3414     glo['stop'] = fn
3415 
3416     fn = {
3417         'each-line': main_each_line,
3418         'each-block': main_each_block,
3419         'all-lines': main_all_lines,
3420         'all-bytes': main_all_bytes,
3421         'json-lines': main_json_lines,
3422         'no-input': main_no_input,
3423         'whole-strings': main_whole_strings,
3424     }[run_mode]
3425 
3426     if fn is None:
3427         raise Exception(f'internal error: invalid run-mode {run_mode}')
3428 
3429     if profile_run:
3430         from cProfile import Profile
3431         # using a profiler in a `with` context adds many irrelevant
3432         # entries to its output
3433         prof = Profile()
3434         prof.enable()
3435         fn(stdout, stdin, expression, inputs)
3436         prof.disable()
3437         prof.print_stats()
3438     else:
3439         fn(stdout, stdin, expression, inputs)
3440 except BrokenPipeError:
3441     # quit quietly, instead of showing a confusing error message
3442     stderr.close()
3443 except KeyboardInterrupt:
3444     exit(2)
3445 except Exception as e:
3446     if trace_exceptions:
3447         raise e
3448     s = str(e)
3449     s = s if s else '<generic exception>'
3450     print(f'\x1b[31m{s}\x1b[0m', file=stderr)
3451     exit(1)