File: jsonl.go 1 /* 2 The MIT License (MIT) 3 4 Copyright (c) 2026 pacman64 5 6 Permission is hereby granted, free of charge, to any person obtaining a copy of 7 this software and associated documentation files (the "Software"), to deal 8 in the Software without restriction, including without limitation the rights to 9 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 10 of the Software, and to permit persons to whom the Software is furnished to do 11 so, subject to the following conditions: 12 13 The above copyright notice and this permission notice shall be included in all 14 copies or substantial portions of the Software. 15 16 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 SOFTWARE. 23 */ 24 25 /* 26 To compile a smaller-sized command-line app, you can use the `go` command as 27 follows: 28 29 go build -ldflags "-s -w" -trimpath jsonl.go 30 */ 31 32 package main 33 34 import ( 35 "bufio" 36 "encoding/json" 37 "errors" 38 "io" 39 "os" 40 ) 41 42 const info = ` 43 jsonl [options...] [filepaths...] 44 45 JSON Lines turns valid JSON-input arrays into separate JSON lines, one for 46 each top-level item. Non-arrays result in a single JSON-line. 47 48 When not given a filepath to load, standard input is used instead. Every 49 output line is always a single top-level item from the input. 50 ` 51 52 func main() { 53 args := os.Args[1:] 54 buffered := false 55 56 for len(args) > 0 { 57 switch args[0] { 58 case `-b`, `--b`, `-buffered`, `--buffered`: 59 buffered = true 60 args = args[1:] 61 continue 62 63 case `-h`, `--h`, `-help`, `--help`: 64 os.Stdout.WriteString(info[1:]) 65 return 66 } 67 68 break 69 } 70 71 if len(args) > 0 && args[0] == `--` { 72 args = args[1:] 73 } 74 75 liveLines := !buffered 76 if !buffered { 77 if _, err := os.Stdout.Seek(0, io.SeekCurrent); err == nil { 78 liveLines = false 79 } 80 } 81 82 if err := run(os.Stdout, args, liveLines); err != nil && err != io.EOF { 83 os.Stderr.WriteString(err.Error()) 84 os.Stderr.WriteString("\n") 85 os.Exit(1) 86 return 87 } 88 } 89 90 func run(w io.Writer, args []string, liveLines bool) error { 91 dashes := 0 92 for _, path := range args { 93 if path == `-` { 94 dashes++ 95 } 96 if dashes > 1 { 97 return errors.New(`can't use stdin (dash) more than once`) 98 } 99 } 100 101 bw := bufio.NewWriter(w) 102 defer bw.Flush() 103 104 if len(args) == 0 { 105 return handleInput(bw, `-`, liveLines) 106 } 107 108 for _, path := range args { 109 if err := handleInput(bw, path, liveLines); err != nil { 110 return err 111 } 112 } 113 return nil 114 } 115 116 // handleInput simplifies control-flow for func main 117 func handleInput(w *bufio.Writer, path string, liveLines bool) error { 118 if path == `-` { 119 return jsonl(w, os.Stdin, liveLines) 120 } 121 122 f, err := os.Open(path) 123 if err != nil { 124 // on windows, file-not-found error messages may mention `CreateFile`, 125 // even when trying to open files in read-only mode 126 return errors.New(`can't open file named ` + path) 127 } 128 defer f.Close() 129 return jsonl(w, f, liveLines) 130 } 131 132 // escapedStringBytes helps funcs like handleString treat all strings 133 // quickly and correctly, using their officially-supported JSON escape 134 // sequences 135 // 136 // https://www.rfc-editor.org/rfc/rfc8259#section-7 137 var escapedStringBytes = [128][]byte{ 138 {'\\', 'u', '0', '0', '0', '0'}, {'\\', 'u', '0', '0', '0', '1'}, 139 {'\\', 'u', '0', '0', '0', '2'}, {'\\', 'u', '0', '0', '0', '3'}, 140 {'\\', 'u', '0', '0', '0', '4'}, {'\\', 'u', '0', '0', '0', '5'}, 141 {'\\', 'u', '0', '0', '0', '6'}, {'\\', 'u', '0', '0', '0', '7'}, 142 {'\\', 'b'}, {'\\', 't'}, 143 {'\\', 'n'}, {'\\', 'u', '0', '0', '0', 'b'}, 144 {'\\', 'f'}, {'\\', 'r'}, 145 {'\\', 'u', '0', '0', '0', 'e'}, {'\\', 'u', '0', '0', '0', 'f'}, 146 {'\\', 'u', '0', '0', '1', '0'}, {'\\', 'u', '0', '0', '1', '1'}, 147 {'\\', 'u', '0', '0', '1', '2'}, {'\\', 'u', '0', '0', '1', '3'}, 148 {'\\', 'u', '0', '0', '1', '4'}, {'\\', 'u', '0', '0', '1', '5'}, 149 {'\\', 'u', '0', '0', '1', '6'}, {'\\', 'u', '0', '0', '1', '7'}, 150 {'\\', 'u', '0', '0', '1', '8'}, {'\\', 'u', '0', '0', '1', '9'}, 151 {'\\', 'u', '0', '0', '1', 'a'}, {'\\', 'u', '0', '0', '1', 'b'}, 152 {'\\', 'u', '0', '0', '1', 'c'}, {'\\', 'u', '0', '0', '1', 'd'}, 153 {'\\', 'u', '0', '0', '1', 'e'}, {'\\', 'u', '0', '0', '1', 'f'}, 154 {32}, {33}, {'\\', '"'}, {35}, {36}, {37}, {38}, {39}, 155 {40}, {41}, {42}, {43}, {44}, {45}, {46}, {47}, 156 {48}, {49}, {50}, {51}, {52}, {53}, {54}, {55}, 157 {56}, {57}, {58}, {59}, {60}, {61}, {62}, {63}, 158 {64}, {65}, {66}, {67}, {68}, {69}, {70}, {71}, 159 {72}, {73}, {74}, {75}, {76}, {77}, {78}, {79}, 160 {80}, {81}, {82}, {83}, {84}, {85}, {86}, {87}, 161 {88}, {89}, {90}, {91}, {'\\', '\\'}, {93}, {94}, {95}, 162 {96}, {97}, {98}, {99}, {100}, {101}, {102}, {103}, 163 {104}, {105}, {106}, {107}, {108}, {109}, {110}, {111}, 164 {112}, {113}, {114}, {115}, {116}, {117}, {118}, {119}, 165 {120}, {121}, {122}, {123}, {124}, {125}, {126}, {127}, 166 } 167 168 // jsonl does it all, given a reader and a writer 169 func jsonl(w *bufio.Writer, r io.Reader, live bool) error { 170 dec := json.NewDecoder(r) 171 // avoid parsing numbers, so unusually-long numbers are kept verbatim, 172 // even if JSON parsers aren't required to guarantee such input-fidelity 173 // for numbers 174 dec.UseNumber() 175 176 t, err := dec.Token() 177 if err == io.EOF { 178 // return errors.New(`input has no JSON values`) 179 return nil 180 } 181 182 if t == json.Delim('[') { 183 if err := handleTopLevelArray(w, dec, live); err != nil { 184 return err 185 } 186 } else { 187 if err := handleToken(w, dec, t); err != nil { 188 return err 189 } 190 w.WriteByte('\n') 191 } 192 193 _, err = dec.Token() 194 if err == io.EOF { 195 // input is over, so it's a success 196 return nil 197 } 198 199 if err == nil { 200 // a successful `read` is a failure, as it means there are 201 // trailing JSON tokens 202 return errors.New(`unexpected trailing data`) 203 } 204 205 // any other error, perhaps some invalid-JSON-syntax-type error 206 return err 207 } 208 209 // handleToken handles recursion for func json2 210 func handleToken(w *bufio.Writer, dec *json.Decoder, t json.Token) error { 211 switch t := t.(type) { 212 case json.Delim: 213 switch t { 214 case json.Delim('['): 215 return handleArray(w, dec) 216 case json.Delim('{'): 217 return handleObject(w, dec) 218 default: 219 return errors.New(`unsupported JSON syntax ` + string(t)) 220 } 221 222 case nil: 223 w.WriteString(`null`) 224 return nil 225 226 case bool: 227 if t { 228 w.WriteString(`true`) 229 } else { 230 w.WriteString(`false`) 231 } 232 return nil 233 234 case json.Number: 235 w.WriteString(t.String()) 236 return nil 237 238 case string: 239 return handleString(w, t) 240 241 default: 242 // return fmt.Errorf(`unsupported token type %T`, t) 243 return errors.New(`invalid JSON token`) 244 } 245 } 246 247 func handleTopLevelArray(w *bufio.Writer, dec *json.Decoder, live bool) error { 248 for i := 0; true; i++ { 249 t, err := dec.Token() 250 if err == io.EOF { 251 return nil 252 } 253 254 if err != nil { 255 return err 256 } 257 258 if t == json.Delim(']') { 259 return nil 260 } 261 262 err = handleToken(w, dec, t) 263 if err != nil { 264 return err 265 } 266 267 if w.WriteByte('\n') != nil { 268 return io.EOF 269 } 270 271 if !live { 272 continue 273 } 274 275 if w.Flush() != nil { 276 return io.EOF 277 } 278 } 279 280 // make the compiler happy 281 return nil 282 } 283 284 // handleArray handles arrays for func handleToken 285 func handleArray(w *bufio.Writer, dec *json.Decoder) error { 286 w.WriteByte('[') 287 288 for i := 0; true; i++ { 289 t, err := dec.Token() 290 if err == io.EOF { 291 return errors.New(`end of JSON before array was closed`) 292 } 293 if err != nil { 294 return err 295 } 296 297 if t == json.Delim(']') { 298 w.WriteByte(']') 299 return nil 300 } 301 302 if i > 0 { 303 _, err := w.WriteString(", ") 304 if err != nil { 305 return io.EOF 306 } 307 } 308 309 err = handleToken(w, dec, t) 310 if err != nil { 311 return err 312 } 313 } 314 315 // make the compiler happy 316 return nil 317 } 318 319 // handleObject handles objects for func handleToken 320 func handleObject(w *bufio.Writer, dec *json.Decoder) error { 321 w.WriteByte('{') 322 323 for i := 0; true; i++ { 324 t, err := dec.Token() 325 if err == io.EOF { 326 return errors.New(`end of JSON before object was closed`) 327 } 328 if err != nil { 329 return err 330 } 331 332 if t == json.Delim('}') { 333 w.WriteByte('}') 334 return nil 335 } 336 337 if i > 0 { 338 _, err := w.WriteString(", ") 339 if err != nil { 340 return io.EOF 341 } 342 } 343 344 k, ok := t.(string) 345 if !ok { 346 return errors.New(`expected a string for a key-value pair`) 347 } 348 349 err = handleString(w, k) 350 if err != nil { 351 return err 352 } 353 354 w.WriteString(": ") 355 356 t, err = dec.Token() 357 if err == io.EOF { 358 return errors.New(`expected a value for a key-value pair`) 359 } 360 361 err = handleToken(w, dec, t) 362 if err != nil { 363 return err 364 } 365 } 366 367 // make the compiler happy 368 return nil 369 } 370 371 // handleString handles strings for func handleToken, and keys for func 372 // handleObject 373 func handleString(w *bufio.Writer, s string) error { 374 for _, r := range s { 375 if r < ' ' || r == '"' || r == '\\' { 376 w.WriteByte('"') 377 for _, r := range s { 378 if int(r) < len(escapedStringBytes) { 379 w.Write(escapedStringBytes[r]) 380 } else { 381 w.WriteRune(r) 382 } 383 } 384 w.WriteByte('"') 385 return nil 386 } 387 } 388 389 w.WriteByte('"') 390 w.WriteString(s) 391 w.WriteByte('"') 392 return nil 393 }