File: gsawk.sh 1 #!/bin/sh 2 3 # The MIT License (MIT) 4 # 5 # Copyright (c) 2026 pacman64 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a copy 8 # of this software and associated documentation files (the "Software"), to deal 9 # in the Software without restriction, including without limitation the rights 10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 # copies of the Software, and to permit persons to whom the Software is 12 # furnished to do so, subject to the following conditions: 13 # 14 # The above copyright notice and this permission notice shall be included in 15 # all copies or substantial portions of the Software. 16 # 17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 # SOFTWARE. 24 25 26 # gsawk [options...] [awk expression...] [files...] 27 # 28 # 29 # Grouped Summaries via AWK expression calculates/aggregates some numeric 30 # statistics for each group determined by the AWK expression given. The 31 # output is a JSON object whose top-level keys are the expression results, 32 # whose values are objects of objects, numerically summarizing all columns 33 # of the rows in its top-level-key group. 34 # 35 # The handy case-insensitive shortcut options may cause this tool to fail, 36 # if the main AWK tool installed doesn't support the special IGNORECASE 37 # variable. 38 # 39 # The AWK options available only in single-dash versions are 40 # 41 # -F fs, -Ffs, -F=fs make `fs` the field separator 42 # 43 # The other options are, available both in single and double-dash versions 44 # 45 # -h, -help show this help message 46 # -i, -ins match regexes case-insensitively; may fail the default `awk` 47 # -sort sort calculated top-level keys 48 # -tsv split fields using tabs, same as using -F "\t" 49 50 51 case "$1" in 52 -h|--h|-help|--help) 53 awk '/^# +gsawk /, /^$/ { gsub(/^# ?/, ""); print }' "$0" 54 exit 0 55 ;; 56 esac 57 58 tsv=0 59 sort_keys='' 60 ins=0 61 cmd='awk' 62 63 while [ $# -gt 0 ]; do 64 case "$1" in 65 -F) 66 if [ $# -lt 2 ]; then 67 printf "expected value after -F option\n" >&2 68 exit 1 69 fi 70 cmd="${cmd} -F $2"; shift 2; continue 71 ;; 72 73 -F*) cmd="${cmd} $1"; shift; continue ;; 74 75 -i|--i|-ins|--ins|-insensitive|--insensitive) ins=1; shift; continue ;; 76 77 -sort|--sort|-sorted|--sorted) 78 sort_keys='asort(keys)'; shift; continue 79 ;; 80 81 -tsv|--tsv) tsv=1; shift; continue ;; 82 83 -v) 84 if [ $# -lt 2 ]; then 85 printf "expected variable assignment after -v option\n" >&2 86 exit 1 87 fi 88 cmd="${cmd} -v $2"; shift 2; continue 89 ;; 90 91 -) break ;; 92 93 --) shift; break ;; 94 95 -*) 96 printf "unsupported option '%s'\n" "$1" >&2 97 exit 1 98 ;; 99 esac 100 101 break 102 done 103 104 code="${1:-\$0}" 105 [ $# -gt 0 ] && shift 106 107 # show all non-existing files given 108 failed=0 109 for arg in "$@"; do 110 [ "${arg}" = "-" ] && continue 111 [ -e "${arg}" ] && continue 112 printf "no file named \"%s\"\n" "${arg}" >&2 113 failed=1 114 done 115 116 [ "${failed}" -gt 0 ] && exit 2 117 118 ci=' 119 BEGIN { 120 if (IGNORECASE == "") { 121 m = "your `awk` command lacks case-insensitive regex-matching" 122 print(m) > "/dev/stderr" 123 exit 125 124 } 125 IGNORECASE = 1 126 } 127 ' 128 if [ "${ins}" -eq 0 ]; then 129 ci='' 130 fi 131 132 src="${ci}"' 133 BEGIN { 134 if (SUBSEP == "") SUBSEP = "\034" 135 inf = "+inf" + 0 136 } 137 138 function stringify(s) { 139 gsub(/\\/, "\\\\", s) 140 gsub(/"/, "\\\"", s) 141 return sprintf("\"%s\"", s) 142 } 143 144 function init_group(key) { 145 summaries[key SUBSEP "numeric"] = 0 146 summaries[key SUBSEP "integer"] = 0 147 summaries[key SUBSEP "positive"] = 0 148 summaries[key SUBSEP "zero"] = 0 149 summaries[key SUBSEP "negative"] = 0 150 summaries[key SUBSEP "min"] = inf 151 summaries[key SUBSEP "max"] = -inf 152 summaries[key SUBSEP "sum"] = 0 153 summaries[key SUBSEP "mean"] = 0 154 summaries[key SUBSEP "product"] = 1 155 156 summaries[key SUBSEP "_ln_sum"] = 0 157 summaries[key SUBSEP "_d1"] = 0 158 summaries[key SUBSEP "_d2"] = 0 159 summaries[key SUBSEP "_mean_square"] = 0 160 } 161 162 function update_group(key, v, n) { 163 summaries[key SUBSEP "numeric"]++ 164 summaries[key SUBSEP "integer"] += v % 1 == 0 165 if (v > 0) summaries[key SUBSEP "positive"]++ 166 else if (v < 0) summaries[key SUBSEP "negative"]++ 167 else if (v == 0) summaries[key SUBSEP "zero"]++ 168 169 n = summaries[key SUBSEP "min"] 170 summaries[key SUBSEP "min"] = n < v ? n : v 171 n = summaries[key SUBSEP "max"] 172 summaries[key SUBSEP "max"] = n > v ? n : v 173 summaries[key SUBSEP "sum"] += v 174 summaries[key SUBSEP "product"] *= v 175 summaries[key SUBSEP "_ln_sum"] += v <= 0 ? -inf : log(v) 176 177 # advance welford`s algorithm 178 n = summaries[key SUBSEP "numeric"] 179 summaries[key SUBSEP "_d1"] = v - summaries[key SUBSEP "mean"] 180 summaries[key SUBSEP "mean"] += summaries[key SUBSEP "_d1"] / n 181 summaries[key SUBSEP "_d2"] = v - summaries[key SUBSEP "mean"] 182 n = summaries[key SUBSEP "_mean_square"] 183 n += summaries[key SUBSEP "_d1"] * summaries[key SUBSEP "_d2"] 184 summaries[key SUBSEP "_mean_square"] = n 185 } 186 187 # ignore leading UTF-8 BOMs on the first line of each input, when present 188 FNR == 1 { gsub(/^\xef\xbb\xbf/, "") } 189 190 # ignore trailing carriage-returns 191 { gsub(/\r$/, "") } 192 193 FNR == 1 { 194 for (i = 1; i <= NF; i++) props[++numprops] = $i 195 next 196 } 197 198 { 199 k = ('"${code}"') 200 201 if (tally[k]++ == 0) { 202 for (i = 1; i <= NF; i++) init_group(k SUBSEP props[i]) 203 keys[++n] = k 204 } 205 206 for (i = 1; i <= NF; i++) { 207 if ($i !~ /^ *(0|[0-9]+|[0-9]*\.[0-9]+) *$/) continue 208 update_group(k SUBSEP props[i], $i + 0) 209 } 210 } 211 212 function emit(key) { 213 nums = summaries[key SUBSEP "numeric"] 214 ints = summaries[key SUBSEP "integer"] 215 pos = summaries[key SUBSEP "positive"] 216 zero = summaries[key SUBSEP "zero"] 217 neg = summaries[key SUBSEP "negative"] 218 min = summaries[key SUBSEP "min"] 219 max = summaries[key SUBSEP "max"] 220 sum = summaries[key SUBSEP "sum"] 221 mean = summaries[key SUBSEP "mean"] 222 product = summaries[key SUBSEP "product"] 223 lsum = summaries[key SUBSEP "_ln_sum"] 224 d1 = summaries[key SUBSEP "_d1"] 225 d2 = summaries[key SUBSEP "_d2"] 226 msq = summaries[key SUBSEP "_mean_square"] 227 sum = mean * nums 228 if (nums == 0) lsum = -inf 229 230 printf "{\"numeric\": %d, ", nums 231 if (nums > 0) { 232 emit_pair("min", min) 233 emit_pair("max", max) 234 emit_pair("sum", sum) 235 emit_pair("mean", mean) 236 gm = "null" 237 if (zero == 0 && neg == 0) gm = sprintf("%f", exp(lsum / nums)) 238 printf "\"geomean\": %s, ", gm 239 emit_pair("sd", sqrt(msq / nums)) 240 } else { 241 printf "\"min\": null, " 242 printf "\"max\": null, " 243 printf "\"sum\": null, " 244 printf "\"mean\": null, " 245 printf "\"geomean\": null, " 246 printf "\"sd\": null, " 247 } 248 printf "\"integer\": %d, ", ints 249 printf "\"positive\": %d, ", pos 250 printf "\"zero\": %d, ", zero 251 printf "\"negative\": %d}", neg 252 } 253 254 function emit_pair(key, num) { 255 printf ((num % 1 == 0) ? "\"%s\": %d, " : "\"%s\": %f, "), key, num 256 } 257 258 END { 259 '"${sort_keys}"' 260 printf "{\n" 261 262 for (i = 1; i <= n; i++) { 263 k1 = keys[i] 264 if (i > 1) printf ",\n" 265 printf " %s: {\n", stringify(k1) 266 267 for (j = 1; j <= numprops; j++) { 268 k2 = props[j] 269 if (j > 1) printf ",\n" 270 printf " %s: ", stringify(k2) 271 emit(k1 SUBSEP k2) 272 } 273 274 if (j > 1) print "" 275 printf " }" 276 } 277 278 if (i > 1) print "" 279 printf "}\n" 280 } 281 ' 282 283 if [ "${tsv}" -eq 1 ]; then 284 ${cmd} -F "\t" "${src}" "$@" 285 else 286 ${cmd} "${src}" "$@" 287 fi