File: gsawk.sh 1 #!/bin/sh 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 # gsawk [options...] [awk expression...] [files...] 27 # 28 # 29 # Grouped Summaries via AWK expression calculates/aggregates some numeric 30 # statistics for each group determined by the AWK expression given. The 31 # output is a JSON object whose top-level keys are the expression results, 32 # whose values are objects of objects, numerically summarizing all columns 33 # of the rows in its top-level-key group. 34 # 35 # The handy case-insensitive shortcut options may cause this tool to fail, 36 # if the main AWK tool installed doesn't support the special IGNORECASE 37 # variable. 38 # 39 # The AWK options available only in single-dash versions are 40 # 41 # -F fs, -Ffs, -F=fs make `fs` the field separator 42 # 43 # The other options are, available both in single and double-dash versions 44 # 45 # -h, -help show this help message 46 # -i, -ins match regexes case-insensitively; may fail the default `awk` 47 # -sort sort calculated top-level keys 48 # -tsv split fields using tabs, same as using -F "\t" 49 50 51 case "$1" in 52 -h|--h|-help|--help) 53 awk '/^# +gsawk /, /^$/ { gsub(/^# ?/, ""); print }' "$0" 54 exit 0 55 ;; 56 esac 57 58 tsv=0 59 sort_keys='' 60 ins=0 61 cmd='awk' 62 63 while [ $# -gt 0 ]; do 64 case "$1" in 65 -F) 66 if [ $# -lt 2 ]; then 67 printf "expected value after -F option\n" >&2 68 exit 1 69 fi 70 cmd="${cmd} -F $2"; shift 2; continue 71 ;; 72 73 -F*) cmd="${cmd} $1"; shift; continue ;; 74 75 -i|--i|-ins|--ins|-insensitive|--insensitive) ins=1; shift; continue ;; 76 77 -sort|--sort|-sorted|--sorted) 78 sort_keys=' asort(KEYS)'; shift; continue 79 ;; 80 81 -tsv|--tsv) tsv=1; shift; continue ;; 82 83 -v) 84 if [ $# -lt 2 ]; then 85 printf "expected variable assignment after -v option\n" >&2 86 exit 1 87 fi 88 cmd="${cmd} -v $2"; shift 2; continue 89 ;; 90 91 -) break ;; 92 93 --) shift; break ;; 94 95 -*) 96 printf "%s: unsupported option '%s'\n" "$0" "$1" >&2 97 exit 1 98 ;; 99 esac 100 101 break 102 done 103 104 code="${1:-\$0}" 105 [ $# -gt 0 ] && shift 106 107 # show all non-existing files given 108 failed=0 109 for arg in "$@"; do 110 [ "${arg}" = "-" ] && continue 111 [ -e "${arg}" ] && continue 112 printf "%s: no file named \"%s\"\n" "$0" "${arg}" >&2 113 failed=1 114 done 115 116 [ "${failed}" -gt 0 ] && exit 2 117 118 ci=' 119 BEGIN { 120 if (IGNORECASE == "") { 121 m = "your `awk` command lacks case-insensitive regex-matching" 122 print(m) > "/dev/stderr" 123 exit 125 124 } 125 IGNORECASE = 1 126 } 127 ' 128 if [ "${ins}" -eq 0 ]; then 129 ci='' 130 fi 131 132 src="${ci}"' 133 BEGIN { 134 if (SUBSEP == "") SUBSEP = "\034" 135 INF = "+inf" + 0 136 } 137 138 function stringify(s) { 139 gsub(/\\/, "\\\\", s) 140 gsub(/"/, "\\\"", s) 141 return sprintf("\"%s\"", s) 142 } 143 144 function init_group(key) { 145 STATS[key SUBSEP "numeric"] = 0 146 STATS[key SUBSEP "integer"] = 0 147 STATS[key SUBSEP "positive"] = 0 148 STATS[key SUBSEP "zero"] = 0 149 STATS[key SUBSEP "negative"] = 0 150 STATS[key SUBSEP "min"] = INF 151 STATS[key SUBSEP "max"] = -INF 152 STATS[key SUBSEP "sum"] = 0 153 STATS[key SUBSEP "mean"] = 0 154 STATS[key SUBSEP "product"] = 1 155 156 STATS[key SUBSEP "_ln_sum"] = 0 157 STATS[key SUBSEP "_d1"] = 0 158 STATS[key SUBSEP "_d2"] = 0 159 STATS[key SUBSEP "_mean_square"] = 0 160 } 161 162 function update_group(key, v, n) { 163 STATS[key SUBSEP "numeric"]++ 164 STATS[key SUBSEP "integer"] += v % 1 == 0 165 if (v > 0) STATS[key SUBSEP "positive"]++ 166 else if (v < 0) STATS[key SUBSEP "negative"]++ 167 else if (v == 0) STATS[key SUBSEP "zero"]++ 168 169 n = STATS[key SUBSEP "min"] 170 STATS[key SUBSEP "min"] = n < v ? n : v 171 n = STATS[key SUBSEP "max"] 172 STATS[key SUBSEP "max"] = n > v ? n : v 173 STATS[key SUBSEP "sum"] += v 174 STATS[key SUBSEP "product"] *= v 175 STATS[key SUBSEP "_ln_sum"] += v <= 0 ? -INF : log(v) 176 177 # advance welford`s algorithm 178 n = STATS[key SUBSEP "numeric"] 179 STATS[key SUBSEP "_d1"] = v - STATS[key SUBSEP "mean"] 180 STATS[key SUBSEP "mean"] += STATS[key SUBSEP "_d1"] / n 181 STATS[key SUBSEP "_d2"] = v - STATS[key SUBSEP "mean"] 182 n = STATS[key SUBSEP "_mean_square"] 183 n += STATS[key SUBSEP "_d1"] * STATS[key SUBSEP "_d2"] 184 STATS[key SUBSEP "_mean_square"] = n 185 } 186 187 # ignore leading UTF-8 BOMs on the first line of each input, when present 188 FNR == 1 { gsub(/^\xef\xbb\xbf/, "") } 189 190 # ignore trailing carriage-returns 191 { gsub(/\r$/, "") } 192 193 FNR == 1 { 194 for (I = 1; I <= NF; I++) PROPS[++NUMPROPS] = $i 195 next 196 } 197 198 { 199 K = ('"${code}"') 200 201 if (TALLY[K]++ == 0) { 202 for (I = 1; I <= NF; I++) init_group(K SUBSEP PROPS[I]) 203 KEYS[++N] = K 204 } 205 206 for (I = 1; I <= NF; I++) { 207 if ($I !~ /^ *(0|[0-9]+|[0-9]*\.[0-9]+) *$/) continue 208 update_group(K SUBSEP PROPS[I], $I + 0) 209 } 210 } 211 212 function emit(key) { 213 NUMS = STATS[key SUBSEP "numeric"] 214 INTS = STATS[key SUBSEP "integer"] 215 POS = STATS[key SUBSEP "positive"] 216 ZERO = STATS[key SUBSEP "zero"] 217 NEG = STATS[key SUBSEP "negative"] 218 MIN = STATS[key SUBSEP "min"] 219 MAX = STATS[key SUBSEP "max"] 220 SUM = STATS[key SUBSEP "sum"] 221 MEAN = STATS[key SUBSEP "mean"] 222 PRODUCT = STATS[key SUBSEP "product"] 223 LSUM = STATS[key SUBSEP "_ln_sum"] 224 D1 = STATS[key SUBSEP "_d1"] 225 D2 = STATS[key SUBSEP "_d2"] 226 MSQ = STATS[key SUBSEP "_mean_square"] 227 SUM = MEAN * NUMS 228 if (NUMS == 0) LSUM = -INF 229 230 printf "{\"numeric\": %d, ", NUMS 231 if (NUMS > 0) { 232 emit_pair("min", MIN) 233 emit_pair("max", MAX) 234 emit_pair("sum", SUM) 235 emit_pair("mean", MEAN) 236 GM = "null" 237 if (ZERO == 0 && NEG == 0) GM = sprintf("%f", exp(lsum / NUMS)) 238 printf "\"geomean\": %s, ", GM 239 emit_pair("sd", sqrt(MSQ / NUMS)) 240 } else { 241 printf "\"min\": null, " 242 printf "\"max\": null, " 243 printf "\"sum\": null, " 244 printf "\"mean\": null, " 245 printf "\"geomean\": null, " 246 printf "\"sd\": null, " 247 } 248 printf "\"integer\": %d, ", INTS 249 printf "\"positive\": %d, ", POS 250 printf "\"zero\": %d, ", ZERO 251 printf "\"negative\": %d}", NEG 252 } 253 254 function emit_pair(key, num) { 255 printf ((num % 1 == 0) ? "\"%s\": %d, " : "\"%s\": %f, "), key, num 256 } 257 258 END { 259 '"${sort_keys}"' 260 printf "{\n" 261 262 for (i = 1; i <= n; i++) { 263 k1 = KEYS[i] 264 if (i > 1) printf ",\n" 265 printf " %s: {\n", stringify(k1) 266 267 for (j = 1; j <= NUMPROPS; j++) { 268 k2 = PROPS[j] 269 if (j > 1) printf ",\n" 270 printf " %s: ", stringify(k2) 271 emit(k1 SUBSEP k2) 272 } 273 274 if (j > 1) print "" 275 printf " }" 276 } 277 278 if (i > 1) print "" 279 printf "}\n" 280 } 281 ' 282 283 if [ "${tsv}" -eq 1 ]; then 284 ${cmd} -F "\t" "${src}" "$@" 285 else 286 ${cmd} "${src}" "$@" 287 fi