File: tj.py 1 #!/usr/bin/python 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 info = ''' 27 tj [options...] [python expression] [filepath/URI...] 28 29 30 Transform Json loads JSON data, runs a Python expression on it, and emits 31 the result as JSON. Parsed input-data are available to the expression as 32 any of the variables named `v`, `value`, `d`, and `data`. 33 34 If no file/URI is given, it loads JSON data from its standard input. If the 35 argument before the expression is a single equals sign (a `=`, without the 36 quotes), no data are read/parsed, and the expression is evaluated as given. 37 38 Options, where leading double-dashes are also allowed, except for alias `=`: 39 40 -c compact single-line JSON output (JSON-0) 41 -compact same as -c 42 -j0 same as -c 43 -json0 same as -c 44 -json-0 same as -c 45 46 -h show this help message 47 -help same as -h 48 49 -nil don't read any input 50 -no-input same as -nil 51 -noinput same as -nil 52 -none same as -nil 53 -null same as -nil 54 -null-input same as -nil 55 -nullinput same as -nil 56 = same as -nil 57 58 -d recursively make dictionary values dot-accessible 59 -dot same as -d 60 -dots same as -d 61 62 -p show a performance/time-profile of the expression run 63 -prof same as -p 64 -profile same as -p 65 66 -t show a full traceback of this script for exceptions 67 -trace same as -t 68 -traceback same as -t 69 70 -z zoom JSON value read from stdin, using all the keys given 71 -zj same as -z 72 -zoom same as -z 73 74 75 Extra Functions 76 77 after(x, y) ignore items until the one given; for strings and sequences 78 afterfinal(x, y) backward counterpart of func after 79 afterlast(x, y) same as func afterfinal 80 arrayish(x) check if value is a list, a tuple, or a generator 81 basename(s) get the final/file part of a pathname 82 before(x, y) ignore items since the one given; for strings and sequences 83 beforefinal(x, y) backward counterpart of func before 84 beforelast(x, y) same as func beforefinal 85 chunk(x, size) split/resequence items into chunks of the length given 86 chunked(x, size) same as func chunk 87 compose(*args) make a func which chain-calls all funcs given 88 composed(*args) same as func compose 89 cond(*args) expression-friendly fully-evaluated if-else chain 90 debase64(s) decode base64 strings, including data-URIs 91 dedup(x) ignore later (re)occurrences of values in a sequence 92 dejson(x, f=None) safe parse JSON from strings 93 denan(x, y) turn a floating-point NaN values into the fallback given 94 denil(*args) return the first non-null/none value among those given 95 denone(*args) same as func denil 96 denull(*args) same as func denil 97 dirname(s) get the folder/directory/parent part of a pathname 98 dive(x, f) transform value in depth-first-recursive fashion 99 divebin(x, y, f) binary (2-input) version of recursive-transform func dive 100 drop(x, *what) ignore keys or substrings; for strings, dicts, dict-lists 101 dropped(x, *v) same as func drop 102 each(x, f) generalization of built-in func map 103 endict(x) turn non-dictionary values into dicts with string keys 104 enfloat(x, f=nan) turn values into floats, offering a fallback on failure 105 enint(x, f=None) turn values into ints, offering a fallback on failure 106 enlist(x) turn non-list values into lists 107 entuple(x) turn non-tuple values into tuples 108 ext(s) return the file-extension part of a pathname, if available 109 fields(s) split fields AWK-style from the string given 110 filtered(x, f) same as func keep 111 flat(*args) flatten everything into an unnested sequence 112 fromto(x, y, ?f) sequence integers, end-value included 113 group(x, ?by) group values into dicts of lists; optional transform func 114 grouped(x, ?by) same as func group 115 harden(f, v) make funcs which return values instead of exceptions 116 hardened(f, v) same as func harden 117 countif(x, f) count how many values make the func given true-like 118 idiota(x, ?f) dict-counterpart of func iota 119 ints(x, y, ?f) make sequences of increasing integers, which include the end 120 iota(x, ?f) make an integer sequence from 1 up to the number given 121 join(x, y) join values into a string; make a dict from keys and values 122 json0(x) turn a value into its smallest JSON-string representation 123 json2(x) turn a value into a 2-space-indented multi-line JSON string 124 jsonl(x) turn a value into a sequence of single-line (JSONL) strings 125 keep(x, pred) generalization of built-in func filter 126 kept(x, pred) same as func keep 127 links(x) auto-detect all hyperlink-like (HTTP/HTTPS) substrings 128 mapped(x, f) same as func each 129 number(x) try to parse as an int, on failure try to parse as a float 130 numbers(x) auto-detect all numbers in the value given 131 numstats(x) calculate various `single-pass` numeric stats 132 once(x, y=None) avoid returning the same value more than once; stateful func 133 pick(x, *what) keep only the keys given; works on dicts, or dict-sequences 134 picked(x, *what) same a func pick 135 plain(s) ignore ANSI-style sequences in strings 136 quoted(s, q='"') surround a string with the (optional) quoting-symbol given 137 recover(*args) recover from exceptions with a fallback value 138 reject(x, pred) generalization of built-in func filter, with opposite logic 139 since(x, y) ignore items before the one given; for strings and sequences 140 sincefinal(x, y) backward counterpart of func since 141 sincelast(x, y) same as func sincefinal 142 split(x, y) split string by separator; split sequence into several ones 143 squeeze(s) strip/trim a string, squishing inner runs of spaces 144 stround(x, d=6) format numbers into decimal-number strings 145 tally(x, ?by) count/tally values, using an optional transformation func 146 tallied(x, ?by) same as func tally 147 trap(x, f=None) try running a func, handing exceptions to a fallback func 148 trycall(*args) same as func recover 149 unique(x) same as func dedup 150 uniqued(x) same as func dedup 151 unjson(x, f=None) same as func dejson 152 unquoted(s) ignore surrounding quotes, if present 153 until(x, y) ignore items after the one given; for strings and sequences 154 untilfinal(x, y) backward counterpart of func until 155 untillast(x, y) same as func untilfinal 156 wait(seconds, x) wait the given number of seconds, before returning a value 157 wat(*args) What Are These (wat) shows help/doc messages for funcs 158 159 160 Examples 161 162 # numbers from 0 to 5; no input is read/used 163 tj = 'range(6)' 164 165 # using bases 1 to 5, find all their powers up to the 4th 166 tj = '((n**p for p in range(1, 4+1)) for n in range(1, 6))' 167 168 # keep only the last 2 items from the input 169 tj = 'range(1, 6)' | tj 'data[-2:]' 170 171 # chunk/regroup input items into arrays of up to 3 items each 172 tj = 'range(1, 8)' | tj 'chunk(data, 3)' 173 174 # ignore all items before the first one with just a 5 in it 175 tj = 'range(8)' | tj 'since(data, 5)' 176 177 # ignore errors/exceptions, in favor of a fallback value 178 tj = 'safe(lambda: 2 * float("no way"), "fallback value")' 179 180 # ignore errors/exceptions, calling a fallback func with the exception 181 tj = 'safe(lambda: 2 * float("no way"), lambda err: str(err))' 182 183 # use dot-syntax on JSON data 184 tj = '{"abc": {"xyz": 123}}' | tj -d 'data.abc.xyz' 185 186 # use dot-syntax on JSON data; keywords as properties are syntax-errors 187 tj = '{"abc": {"def": 123}}' | tj -d 'data.abc["def"]' 188 189 # func results are automatically called on the input 190 tj = '{"abc": 123, "def": 456}' | tj len 191 ''' 192 193 194 from sys import argv, exit, stderr, stdin, stdout 195 196 197 if __name__ != '__main__': 198 print('don\'t import this script, run it directly instead', file=stderr) 199 exit(1) 200 201 # no args or a leading help-option arg means show the help message and quit 202 help_opts = ('-h', '--h', '-help', '--help') 203 if len(argv) < 2 or (len(argv) == 2 and argv[1] in help_opts): 204 print(info.strip(), file=stderr) 205 exit(0) 206 207 208 from io import StringIO 209 from itertools import islice 210 from json import load 211 212 from typing import \ 213 AbstractSet, Annotated, Any, AnyStr, \ 214 AsyncContextManager, AsyncGenerator, AsyncIterable, AsyncIterator, \ 215 Awaitable, BinaryIO, ByteString, Callable, cast, \ 216 ClassVar, Collection, Container, \ 217 ContextManager, Coroutine, Deque, Dict, Final, \ 218 final, ForwardRef, FrozenSet, Generator, Generic, get_args, get_origin, \ 219 get_type_hints, Hashable, IO, ItemsView, \ 220 Iterable, Iterator, KeysView, List, Literal, Mapping, \ 221 MappingView, Match, MutableMapping, MutableSequence, MutableSet, \ 222 NamedTuple, NewType, no_type_check, no_type_check_decorator, \ 223 NoReturn, Optional, overload, \ 224 Protocol, Reversible, \ 225 runtime_checkable, Sequence, Set, Sized, SupportsAbs, \ 226 SupportsBytes, SupportsComplex, SupportsFloat, SupportsIndex, \ 227 SupportsInt, SupportsRound, Text, TextIO, Tuple, Type, \ 228 TypedDict, TypeVar, \ 229 TYPE_CHECKING, Union, ValuesView 230 try: 231 from typing import \ 232 assert_never, assert_type, clear_overloads, Concatenate, \ 233 dataclass_transform, get_overloads, is_typeddict, LiteralString, \ 234 Never, NotRequired, ParamSpec, ParamSpecArgs, ParamSpecKwargs, \ 235 Required, reveal_type, Self, TypeAlias, TypeGuard, TypeVarTuple, \ 236 Unpack 237 from typing import \ 238 AwaitableGenerator, override, TypeAliasType, type_check_only 239 except Exception as _: 240 pass 241 242 243 def conforms(x: Any) -> bool: 244 ''' 245 Check if a value is JSON-compatible, which includes checking values 246 recursively, in case of composite/nestable values. 247 ''' 248 249 if x is None or isinstance(x, (bool, int, str)): 250 return True 251 if isinstance(x, float): 252 return not (isnan(x) or isinf(x)) 253 if isinstance(x, (list, tuple)): 254 return all(conforms(e) for e in x) 255 if isinstance(x, dict): 256 return all(conforms(k) and conforms(v) for k, v in x.items()) 257 return False 258 259 260 def seems_url(s: str) -> bool: 261 protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:') 262 return any(s.startswith(p) for p in protocols) 263 264 265 def result_needs_fixing(x: Any) -> bool: 266 ''' 267 See if func fix_result needs to be called: avoiding that can speed 268 things up and save memory, when a composite value is big enough. 269 ''' 270 271 if x is None or isinstance(x, (bool, int, float, str)): 272 return False 273 rec = result_needs_fixing 274 if isinstance(x, dict): 275 return any(rec(k) or rec(v) for k, v in x.items()) 276 if isinstance(x, (list, tuple)): 277 return any(rec(e) for e in x) 278 return True 279 280 281 def fix_result(x: Any, default: Any) -> Any: 282 'Adapt a value so it can be output.' 283 284 if x is type: 285 return type(default).__name__ 286 287 # if expression results in a func, auto-call it with the original data 288 if callable(x): 289 c = required_arg_count(x) 290 if c == 1: 291 x = x(default) 292 else: 293 m = f'func auto-call only works with 1-arg funcs (func wanted {c})' 294 raise Exception(m) 295 296 if x is None or isinstance(x, (bool, int, float, str)): 297 return x 298 299 rec = fix_result 300 301 if isinstance(x, dict): 302 return { 303 rec(k, default): rec(v, default) for k, v in x.items() if not 304 (isinstance(k, Skip) or isinstance(v, Skip)) 305 } 306 if isinstance(x, Iterable): 307 return tuple(rec(e, default) for e in x if not isinstance(e, Skip)) 308 309 if isinstance(x, Dottable): 310 return rec(x.__dict__, default) 311 if isinstance(x, DotCallable): 312 return rec(x.value, default) 313 314 if isinstance(x, Exception): 315 raise x 316 317 return None if isinstance(x, Skip) else str(x) 318 319 320 def disabled_exec(*args, **kwargs) -> None: 321 _ = args 322 _ = kwargs 323 raise Exception('built-in func `exec` is disabled') 324 325 326 def disabled_open(*args, **kwargs) -> None: 327 _ = args 328 _ = kwargs 329 raise Exception('built-in func `open` is disabled')