File: dedup.py
   1 #!/usr/bin/python3
   2 
   3 # The MIT License (MIT)
   4 #
   5 # Copyright © 2020-2025 pacman64
   6 #
   7 # Permission is hereby granted, free of charge, to any person obtaining a copy
   8 # of this software and associated documentation files (the “Software”), to deal
   9 # in the Software without restriction, including without limitation the rights
  10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11 # copies of the Software, and to permit persons to whom the Software is
  12 # furnished to do so, subject to the following conditions:
  13 #
  14 # The above copyright notice and this permission notice shall be included in
  15 # all copies or substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23 # SOFTWARE.
  24 
  25 
  26 from io import SEEK_CUR
  27 from sys import argv, exit, stderr, stdin, stdout
  28 from typing import List, Set
  29 
  30 
  31 info = '''
  32 dedup [options...] [filepaths/URIs...]
  33 
  34 This script reads/fetches all named sources given to it, and only outputs
  35 each exact same line once, deduplicating lines. Named sources can be a mix
  36 of filepaths and URIs.
  37 
  38 Each non-empty input ends with a line feed, even when the original data
  39 don't.
  40 
  41 The help option is `-h`, `--h`, `-help`, or `--help`.
  42 '''
  43 
  44 # a leading help-option arg means show the help message and quit
  45 if len(argv) > 1 and argv[1] in ('-h', '--h', '-help', '--help'):
  46     print(info.strip())
  47     exit(0)
  48 
  49 
  50 def seems_url(s: str) -> bool:
  51     protocols = ('https://', 'http://', 'file://', 'ftp://', 'data:')
  52     return any(s.startswith(p) for p in protocols)
  53 
  54 
  55 def run(args: List[str]) -> None:
  56     try:
  57         stdout.seek(0, SEEK_CUR)
  58         live = False
  59     except:
  60         live = True
  61 
  62     if any(seems_url(e) for e in args):
  63         from io import TextIOWrapper
  64         from urllib.request import urlopen
  65 
  66     paths = set()
  67     lines = set()
  68 
  69     for path in args:
  70         if path in paths:
  71             continue
  72         paths.add(path)
  73 
  74         if path == '-':
  75             handle_lines(stdout, inp, lines, live)
  76             continue
  77 
  78         if seems_url(path):
  79             with urlopen(path) as inp:
  80                 with TextIOWrapper(inp, encoding='utf-8') as txt:
  81                     handle_lines(stdout, txt, lines, live)
  82             continue
  83 
  84         with open(path, encoding='utf-8') as inp:
  85             handle_lines(stdout, inp, lines, live)
  86 
  87     if len(args) == 0:
  88         handle_lines(stdout, stdin, lines, live)
  89 
  90 
  91 def handle_lines(w, src, got: Set[str], live: bool) -> None:
  92     for line in src:
  93         line = line.rstrip('\n')
  94         if line in got:
  95             continue
  96         got.add(line)
  97 
  98         w.write(line)
  99         w.write('\n')
 100         if live:
 101             w.flush()
 102 
 103 
 104 try:
 105     if len(argv) > 1 and argv[1] == '--':
 106         run(argv[2:])
 107     else:
 108         run(argv[1:])
 109 except KeyboardInterrupt:
 110     exit(2)
 111 except Exception as e:
 112     print(f'\x1b[31m{e}\x1b[0m', file=stderr)
 113     exit(1)