#!/usr/bin/python

# The MIT License (MIT)
#
# Copyright (c) 2026 pacman64
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


info = '''
groupy [options...] [python expression] [files/URIs...]


GROUp with PYthon groups/ties lines together by running the Python expression
given on each line, and using its common results. Inputs are encoded as UTF-8.
Carriage-returns are always ignored in lines, as well as any UTF-8-BOM on the
first line of each input.

The expression can use either `l` or `line` for the current line, and `i` as
a 0-based line counter which keeps growing even across input-sources, when
given multiple inputs. Also available is `n`, a 1-based line counter which
otherwise works the same way.

Each line is automatically parsed as JSON: when successful, the parsed line
is available to the expression as `v`, or `value`. You can check failure to
parse JSON by checking if `v` or `value` is of type Skip, since None can be
the result of successfully parsing a null JSON value.

Input-sources can be either files or web-URIs. When not given any explicit
named sources, the standard input is used. A single dash also means the
standard input.

When the expression results in None, the current input line is ignored.


Options

All options can start with either a single or a double leading dash:

    -c, -compact, -j0, -json0      emit a single-line of compact JSON instead
    -h, -help                      show this help message
    -j, -json                      emit multi-line JSON, instead of lines
    -m, -mod, -module, -modules    import modules named in the next argument,
                                   where multiple names are comma-separated
    -t, -trace, -traceback         turn exceptions into multi-line tracebacks


Examples

# read from a file, and only emit lines exceeding 80 items
groupy '1 if len(line) > 80 else None' data.txt

# group files in current folder (roughly) by common auto-detected file-type
file * | groupy '(lambda e: e[1] + e[2])(" ".join(l.split(":")[1:]).split())'

# group files in current folder (roughly) by common auto-detected file-type
file * | groupy 'fields[1] if len(fields) < 3 else fields[1] + fields[2]'

# group files by auto-detected file-type, emitting JSON output instead
file * | groupy -j 'items[1] if len(items) < 3 else f"{items[1]} {items[2]}"'

# group numbers by common remainder when divided by 5
seq 50 | groupy 'v % 5'

# transform lines in order with a format-string, without grouping anything
seq 50 | groupy 'print(f"{line + "**2":>6} = {float(v**2):12,.4f}")'

# group numbers by common remainder when divided by 5, ignoring numbers > 30
seq 50 | groupy 'None if v > 30 else v % 5'

# alternate lines into 3 rotating groups, limiting each group-size to 5
seq 500 | groupy 'n % 3 if len(groups.get(n % 3, [])) < 5 else None'
'''


from itertools import islice, zip_longest
from json import dump, dumps, loads
from math import isinf, isnan
from re import compile as compile_uncached, IGNORECASE
from sys import argv, exit, stderr, stdin, stdout
from time import localtime, strftime
from typing import Iterable


if len(argv) < 2:
    print(info.strip(), file=stderr)
    exit(1)
if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'):
    print(info.strip())
    exit(0)


def group_lines(src, expr):
    global first_key

    # `comprehension` expressions seem to ignore local variables: even
    # lambda-based workaround-attempts fail to make needed values like
    # the current line available to such expressions
    global i, n, l, line, v, val, value
    global groups, items, fields, words, tsv

    for l in src:
        l = l.rstrip('\r\n').rstrip('\n')
        if i == 0:
            l = l.lstrip('\xef\xbb\xbf')

        line = l
        items = fields = words = line.split()
        tsv = line.split('\t')

        try:
            v = val = value = loads(l)
        except Exception as _:
            v = val = value = Skip()

        k = eval(expr)
        if callable(k):
            k = k(line)
        if isinstance(k, BaseException):
            raise k

        i += 1
        n += 1

        if k is None or isinstance(k, Skip):
            continue
        if first_key is None:
            first_key = k

        if k in groups:
            groups[k].append(line)
        else:
            groups[k] = [line]

        # emit lines for the first group right away, since its items are
        # available as soon as they come from the input
        if k == first_key:
            print(line, flush=flushed)

def fail(msg, code = 1):
    print(str(msg), file=stderr)
    exit(code)

def make_open_utf8(open):
    def open_utf8_readonly(path):
        return open(path, encoding='utf-8')
    return open_utf8_readonly

def seemsurl(path):
    protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:')
    return any(path.startswith(p) for p in protocols)

class Skip:
    def __call__(self, x):
        return isinstance(x, self.__class__)

skip = Skip()

def chunk(items, chunk_size):
    'Break iterable into chunks, each with up to the item-count given.'

    if isinstance(items, str):
        n = len(items)
        while n >= chunk_size:
            yield items[:chunk_size]
            items = items[chunk_size:]
            n -= chunk_size
        if n > 0:
            yield items
        return

    if not isinstance(chunk_size, int):
        raise Exception('non-integer chunk-size')
    if chunk_size < 1:
        raise Exception('non-positive chunk-size')

    it = iter(items)
    while True:
        head = tuple(islice(it, chunk_size))
        if not head:
            return
        yield head

chunked = chunk

# re_cache is used by custom func compile to cache previously-compiled
# regular-expressions, which makes them quicker to (re)use in formulas
re_cache = {}

def re_compile(expr, flags = 0):
    'Speed-up using regexes across lines, by avoiding recompilations.'

    if flags in re_cache:
        cache = re_cache[flags]
    else:
        cache = {}
        re_cache[flags] = cache
    if expr in cache:
        return cache[expr]

    pat = compile_uncached(expr, flags)
    cache[expr] = pat
    return pat

def icompile(expr):
    return re_compile(expr, IGNORECASE)

def cond(*args):
    if len(args) == 0:
        return None

    for i, e in enumerate(args):
        if i % 2 == 0 and i < len(args) - 1 and e:
            return args[i + 1]

    return args[-1] if len(args) % 2 == 1 else None

def dive(into, using):
    'Depth-first recursive caller for 1-input functions.'

    if callable(into):
        into, using = using, into

    def rec(v):
        if isinstance(v, dict):
            return {k: rec(v) for k, v in v.items()}
        if isinstance(v, Iterable) and not isinstance(v, str):
            return [rec(v) for v in v]
        return using(v)

    return rec(into)

def divekeys(into, using):
    'Depth-first recursive caller for 2-input funcs which rename dict keys.'

    if callable(into):
        into, using = using, into

    def rec(v):
        if isinstance(v, dict):
            return {using(k): rec(v) for k, v in v.items()}
        if isinstance(v, Iterable) and not isinstance(v, str):
            return [rec(v) for i, v in enumerate(v)]
        return v

    return rec(None, into)

def divekv(into, using, using2 = None):
    'Depth-first recursive caller for 2-input functions.'

    if using2 is None:
        if callable(into):
            into, using = using, into
    else:
        if not callable(using2):
            into, using, using2 = using2, into, using

    def rec(k, v):
        if isinstance(v, dict):
            return {k: rec(k, v) for k, v in v.items()}
        if isinstance(v, Iterable) and not isinstance(v, str):
            return [rec(i, v) for i, v in enumerate(v)]
        return using(k, v)

    def rec2(k, v):
        if isinstance(v, dict):
            return {str(using(k, v)): rec2(k, v) for k, v in v.items()}
        if isinstance(v, Iterable) and not isinstance(v, str):
            return [rec2(i, v) for i, v in enumerate(v)]
        return using2(k, v)

    return rec(None, into) if using2 is None else rec2(None, into)

kvdive = divekv

def drop(src, *what):
    if isinstance(src, str):
        for s in what:
            src = src.replace(s, '')
        return src

    def kdrop(src, what):
        return {k: v for (k, v) in src.items() if not (k in what)}

    if isinstance(src, dict):
        return kdrop(src, set(what))

    if isinstance(src, Iterable):
        what = set(what)
        return [kdrop(e, what) for e in src if isinstance(e, dict)]

    return None

dropped = drop

def join(x, *y):
    'Join values into a string, or make a dict from keys and values.'

    if len(y) == 0:
        return ' '.join(str(v) for v in x)
    if isinstance(x, str):
        return x.join(str(v) for v in y)
    if len(y) == 1 and isinstance(y[0], str):
        return y[0].join(str(v) for v in x)
    if len(y) == 1 and isinstance(y[0], (list, range, set, tuple, Generator)):
        return {k: v for k, v in zip_longest(x, y[0]) if not (k is None)}
    if isinstance(y, (list, range, set, tuple, Generator)):
        return {k: v for k, v in zip_longest(x, y) if not (k is None)}
    return {k: y for k in x}

def maybe(f, x):
    try:
        return f(x)
    except Exception as _:
        return x

def number(x):
    try:
        return int(x)
    except Exception as _:
        pass
    try:
        return float(x)
    except Exception as _:
        return x

def pick(src, *keys):
    if isinstance(src, dict):
        return {k: src.get(k, None) for k in keys}
    return [{k: e.get(k, None) for k in keys} for e in src if isinstance(e, dict)]

picked = pick

def plain(s):
    'Ignore all ANSI-style sequences in a string.'
    return re_compile('''\x1b\\[([0-9;]+m|[0-9]*[A-HJKST])''').sub('', s)

def predicate(x):
    'Helps various higher-order funcs, by standardizing `predicate` values.'
    if callable(x):
        return x
    if isinstance(x, float):
        if isnan(x):
            return lambda y: isinstance(y, float) and isnan(y)
        if isinf(x):
            return lambda y: isinstance(y, float) and isinf(y)
    return lambda y: x == y

def rescue(attempt, fallback = None):
    try:
        return attempt()
    except Exception as e:
        if callable(fallback):
            return fallback(e)
        return fallback

rescued = rescue

def retype(x):
    'Try to narrow the type of the value given.'

    if isinstance(x, float):
        n = int(x)
        return n if float(n) == x else x

    if not isinstance(x, str):
        return x

    try:
        return loads(x)
    except Exception:
        pass

    try:
        return int(x)
    except Exception:
        pass

    try:
        return float(x)
    except Exception:
        pass

    return x

autocast = autocasted = mold = molded = recast = recasted = remold = retype
remolded = retyped = retype

def json0(x):
    if isinstance(x, (range, set, Generator)):
        x = tuple(x)
    return dumps(x, separators=(',', ':'), allow_nan=False, indent=None)

j0 = json0

def typeof(x):
    return {
        type(None): 'null',
        bool: 'boolean',
        dict: 'object',
        float: 'number',
        int: 'number',
        str: 'string',
        list: 'array',
        tuple: 'array',
    }.get(type(x), 'other')

jstype = typeof

def after(x, what):
    i = x.find(what)
    return '' if i < 0 else x[i+len(what):]

def afterlast(x, what):
    i = x.rfind(what)
    return '' if i < 0 else x[i+len(what):]

afterfinal = afterlast

def before(x, what):
    i = x.find(what)
    return x if i < 0 else x[:i]

def beforelast(x, what):
    i = x.rfind(what)
    return x if i < 0 else x[:i]

beforefinal = beforelast

def since(x, what):
    i = x.find(what)
    return '' if i < 0 else x[i:]

def sincelast(x, what):
    i = x.rfind(what)
    return '' if i < 0 else x[i:]

sincefinal = sincelast

def until(x, what):
    i = x.find(what)
    return x if i < 0 else x[:i+len(what)]

def untilfinal(x, what):
    i = x.rfind(what)
    return x if i < 0 else x[:i+len(what)]

untillast = untilfinal

def message(msg, result = None):
    print(msg, file=stderr)
    return result

msg = message

# seen is used by func `once` to remember previously-given values
seen = set()

def once(x):
    if x in seen:
        return None
    seen.add(x)
    return x

dedup = unique = once

def utf8(x):
    try:
        if isinstance(x, str):
            x = x.encode('utf-8')
        return str(x, 'utf-8')
    except Exception:
        return None

def ymdhms(when = None):
    fmt = f'%Y-%m-%d %H:%M:%S'
    if isinstance(when, (float, int)):
        return strftime(fmt, localtime(float(when)))
    if isinstance(when, tuple):
        return strftime(fmt, when)
    return strftime(fmt, localtime())


dquo = dquote = '"'
lcurly = '{'
rcurly = '}'
squo = squote = '\''

nil = none = null = None


exec = None
open_utf8 = make_open_utf8(open)
open = open_utf8

json_opts = ('-j', '--j', '-json', '--json')
compact_json_opts = (
    '-c', '--c', '-compact', '--compact',
    '-j0', '--j0', '-json0', '--json0', '-json-0', '--json-0',
)
modules_opts = (
    '-m', '--m', '-mod', '--mod', '-module', '--module',
    '-modules', '--modules',
)
trace_opts = ('-t', '--t', '-trace', '--trace', '-traceback', '--traceback')

args = argv[1:]
if any(seemsurl(e) for e in args):
    from io import TextIOWrapper
    from urllib.request import urlopen

emit_json = False
compact_json = False
trace_errors = False

while len(args) > 0:
    if args[0] == '--':
        args = args[1:]
        break

    if args[0] in modules_opts:
        try:
            if len(args) < 2:
                msg = 'a module name or a comma-separated list of modules'
                raise Exception('expected ' + msg)

            g = globals()
            from importlib import import_module
            for e in args[1].split(','):
                g[e] = import_module(e)

            g = None
            import_module = None
            args = args[2:]
        except Exception as e:
            fail(e, 1)

        continue

    if args[0] in json_opts:
        emit_json = True
        args = args[1:]
        continue

    if args[0] in compact_json_opts:
        compact_json = True
        args = args[1:]
        continue

    if args[0] in trace_opts:
        trace_errors = True
        args = args[1:]
        continue

    break


# ensure live-lines output, unless stdout is being saved into a file
flushed = stdout.isatty() or not stdout.seekable()

first_key = None

expr = '.'
if len(args) > 0:
    expr = args[0]
    args = args[1:]

if expr == '.':
    expr = 'line'

try:
    expr = compile(expr, expr, mode='eval')
    compile = None

    i = 0
    n = 1
    v = val = value = Skip()

    groups = {}
    items = fields = words = []
    tsv = []

    if len(args) == 0:
        group_lines(stdin, expr)

    if args.count('-') > 1:
        msg = 'reading from `-` (standard input) more than once not allowed'
        raise ValueError(msg)

    for path in args:
        if path == '-':
            group_lines(stdin, expr)
            continue

        if seemsurl(path):
            with urlopen(path) as inp:
                with TextIOWrapper(inp, encoding='utf-8') as txt:
                    group_lines(txt, expr)
            continue

        with open_utf8(path) as txt:
            group_lines(txt, expr)

    if compact_json:
        dump(groups, stdout, check_circular=False, allow_nan=False)
        print()
    elif emit_json:
        dump(groups, stdout, check_circular=False, allow_nan=False,
             separators=(', ', ': '), indent=4)
        print()
    else:
        for i, group in enumerate(groups.values()):
            # all items from the first key/group are already output
            if i == 0:
                continue
            for e in group:
                print(e)
except BrokenPipeError:
    # quit quietly, instead of showing a confusing error message
    stderr.close()
    exit(0)
except KeyboardInterrupt:
    exit(2)
except Exception as e:
    if trace_errors:
        raise e
    else:
        fail(e, 1)