#!/usr/bin/python3

# The MIT License (MIT)
#
# Copyright © 2020-2025 pacman64
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


from sys import argv, exit, stderr, stdin, stdout
from typing import List, Set


info = '''
dedup [options...] [filepaths/URIs...]

This script reads/fetches all named sources given to it, and only outputs
each exact same line once, deduplicating lines. Named sources can be a mix
of filepaths and URIs.

Each non-empty input ends with a line feed, even when the original data
don't.

The help option is `-h`, `--h`, `-help`, or `--help`.
'''

# a leading help-option arg means show the help message and quit
if len(argv) == 2 and argv[1] in ('-h', '--h', '-help', '--help'):
    print(info.strip())
    exit(0)


def seems_url(s: str) -> bool:
    protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:')
    return any(s.startswith(p) for p in protocols)


def run(args: List[str]) -> None:
    paths = set()
    lines = set()

    if any(seems_url(e) for e in args):
        from io import TextIOWrapper
        from urllib.request import urlopen

    for path in args:
        if path in paths:
            continue
        paths.add(path)

        if path == '-':
            handle_lines(stdout, inp, lines)
            continue

        if seems_url(path):
            with urlopen(path) as inp:
                with TextIOWrapper(inp, encoding='utf-8') as txt:
                    handle_lines(stdout, txt, lines)
            continue

        with open(path, encoding='utf-8') as inp:
            handle_lines(stdout, inp, lines)

    if len(args) == 0:
        handle_lines(stdout, stdin, lines)


def handle_lines(w, src, got: Set[str]) -> None:
    for line in src:
        if line in got:
            continue
        got.add(line)

        w.write(line)
        if not line.endswith('\n'):
            w.write('\n')


try:
    run(argv[1:])
except BrokenPipeError:
    # quit quietly, instead of showing a confusing error message
    stderr.close()
except KeyboardInterrupt:
    exit(2)
except Exception as e:
    print(f'\x1b[31m{e}\x1b[0m', file=stderr)
    exit(1)