diff --git a/TODO.md b/TODO.md index 09fe978..e6f2108 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,7 @@ - [ ] git — common workflows, rebase, stash, bisect - [x] ansible — playbook patterns, inventory, vault, variables, roles - [ ] podman — build, run, compose, volumes -- [ ] jq — filters, select, map, slurp +- [x] jq — filters, select, map, slurp, recipes, @format - [ ] curl — headers, auth, methods, output - [ ] systemd — units, journalctl, timers - [ ] ssh — config, tunnels, keys, agent diff --git a/topics/jq.md b/topics/jq.md new file mode 100644 index 0000000..cab34b6 --- /dev/null +++ b/topics/jq.md @@ -0,0 +1,275 @@ +# jq + +> Lightweight command-line JSON processor — `sed` for structured data. + +## Basics + +```bash +# Pretty-print +echo '{"a":1}' | jq '.' + +# Read from file +jq '.' data.json + +# Compact output (no whitespace) +jq -c '.' data.json + +# Raw string output (no quotes) +jq -r '.name' data.json + +# Raw input (treat each line as string) +jq -R '.' <<< "hello" + +# Null input (build JSON from scratch) +jq -n '{name: "test", version: 1}' + +# Slurp — read all inputs into one array +jq -s '.' file1.json file2.json + +# Pass variables from shell +jq --arg name "$USER" '.user = $name' data.json +jq --argjson count 5 '.limit = $count' data.json +``` + +## Common Flags + +| Flag | Effect | +|---------------|---------------------------------------------| +| `-r` | Raw output (strings without quotes) | +| `-c` | Compact (one line per object) | +| `-s` | Slurp all inputs into array | +| `-n` | Null input (don't read stdin) | +| `-e` | Exit 1 if last output is `false` or `null` | +| `-R` | Raw input (lines as strings, not JSON) | +| `-S` | Sort object keys | +| `--arg k v` | Bind string `$k` to value `v` | +| `--argjson k v`| Bind `$k` to JSON value `v` | +| `--slurpfile k f` | Bind `$k` to array of JSON values from file | +| `--rawfile k f` | Bind `$k` to string contents of file | +| `--tab` | Indent with tabs | +| `--indent N` | Indent with N spaces (default: 2) | + +## Navigation + +```bash +# Object access +jq '.name' # field +jq '.address.city' # nested +jq '.["hyphen-key"]' # keys with special chars + +# Array access +jq '.[0]' # first element +jq '.[-1]' # last element +jq '.[2:5]' # slice (index 2,3,4) +jq '.[:3]' # first 3 +jq '.[-2:]' # last 2 + +# Iteration +jq '.[]' # all array elements / object values +jq '.users[]' # iterate array field +jq '.users[].name' # pluck field from each + +# Optional access (no error if missing) +jq '.maybe?.nested?.field' +``` + +## Constructing Output + +```bash +# Build objects +jq '{name: .user, id: .uid}' + +# Build arrays +jq '[.items[].name]' + +# String interpolation +jq -r '"User: \(.name) (age \(.age))"' + +# Multiple outputs (one per line) +jq '.name, .age' + +# Pipe within expression +jq '.users[] | {name, email}' +``` + +## Filtering and Selection + +```bash +# Select by condition +jq '.[] | select(.age > 30)' +jq '.[] | select(.name == "alice")' +jq '.[] | select(.tags | contains(["prod"]))' +jq '.[] | select(.name | test("^web"))' # regex match +jq '.[] | select(.status | IN("active","pending"))' # multiple values + +# Limit results +jq '[.[] | select(.active)][:5]' # first 5 matches +jq 'first(.[] | select(.ready))' # first match only +jq 'limit(3; .[] | select(.ready))' # first 3 matches + +# Check existence +jq '.[] | select(has("email"))' +jq '.[] | select(.phone != null)' +``` + +## Transformation + +```bash +# Map — transform each element +jq '[.[] | .name]' # pluck +jq 'map(.price * .quantity)' # compute +jq 'map(select(.active))' # filter +jq 'map({name, upper: (.name | ascii_upcase)})' # reshape + +# Sort +jq 'sort_by(.name)' +jq 'sort_by(.date) | reverse' + +# Group +jq 'group_by(.department)' +jq 'group_by(.type) | map({type: .[0].type, count: length})' + +# Unique +jq '[.[] .category] | unique' +jq 'unique_by(.email)' + +# Flatten +jq '[.teams[].members[]] | flatten' + +# Length / count +jq '.items | length' +jq '[.[] | select(.active)] | length' + +# Min / max +jq 'min_by(.price)' +jq '[.[] .score] | max' + +# Reduce — fold into single value +jq 'reduce .[] as $x (0; . + $x.amount)' +``` + +## String Operations + +```bash +jq '.name | ascii_downcase' +jq '.name | ascii_upcase' +jq '.name | ltrimstr("prefix_")' +jq '.name | rtrimstr("_suffix")' +jq '.line | split(",")' # string -> array +jq '.tags | join(", ")' # array -> string +jq '.desc | gsub("old"; "new")' # replace all +jq '.line | test("^[0-9]+$")' # regex test (boolean) +jq '.line | capture("(?\\w+)=(?\\w+)")' # named captures -> object +jq '.name | length' # string length +jq '"hello" + " " + "world"' # concatenation +``` + +## Type Operations + +```bash +jq '.value | type' # "string", "number", "object", etc. +jq '.[] | select(type == "object")' +jq '.value | tostring' +jq '.str_num | tonumber' +jq 'null // "default"' # alternative operator (fallback) +jq '.missing // empty' # suppress null output +jq 'if .count > 0 then "yes" else "no" end' +``` + +## Object Manipulation + +```bash +# Add / update field +jq '.enabled = true' +jq '.tags += ["new"]' +jq '.config.timeout = 30' + +# Remove field +jq 'del(.temporary)' +jq 'del(.users[0])' + +# Rename key +jq '.new_name = .old_name | del(.old_name)' + +# Merge objects +jq '. + {"extra": true}' +jq '. * {"nested": {"override": 1}}' # recursive merge + +# Get keys / values +jq 'keys' +jq 'values' +jq 'to_entries' # [{key, value}, ...] +jq 'from_entries' # reverse +jq 'to_entries | map(.key)' # same as keys + +# Filter object by keys +jq '{name, email}' # keep only these +jq 'with_entries(select(.key | test("^app_")))' # keys matching pattern +``` + +## Practical Recipes + +```bash +# CSV-like output from JSON array +jq -r '.[] | [.name, .email, .role] | @csv' + +# TSV output +jq -r '.[] | [.name, .age] | @tsv' + +# JSON array -> newline-delimited JSON (NDJSON) +jq -c '.[]' array.json + +# NDJSON -> JSON array +jq -s '.' stream.jsonl + +# Merge multiple JSON files +jq -s 'add' file1.json file2.json + +# Deep merge +jq -s '.[0] * .[1]' base.json override.json + +# Pivot: array of objects -> lookup object +jq 'map({(.id | tostring): .}) | add' + +# Frequency count +jq 'group_by(.status) | map({status: .[0].status, count: length})' + +# Update nested value conditionally +jq '(.items[] | select(.id == 42)).status = "done"' + +# Format as env vars +jq -r 'to_entries[] | "\(.key)=\(.value)"' + +# Pretty-print curl JSON response +curl -s https://api.example.com/data | jq '.' +``` + +## @format Strings + +```bash +jq -r '@csv' # CSV encoding +jq -r '@tsv' # TSV encoding +jq -r '@html' # HTML entity escaping +jq -r '@uri' # Percent-encoding +jq -r '@base64' # Base64 encode +jq -r '@base64d' # Base64 decode +jq -r '@json' # JSON encode (escape string as JSON) +jq -r '@text' # Identity (useful in interpolation) +jq -r '@sh' # Shell-escaped string +``` + +## Gotchas + +- Bare `jq '.'` on non-JSON input produces a parse error — use `-R` for raw lines +- `--arg` always passes a **string**; use `--argjson` for numbers/booleans/null +- `.foo` on an array fails — use `.[] .foo` or `map(.foo)` to iterate first +- `select()` that matches nothing produces **no output**, not `null` +- `//` (alternative) triggers on both `null` and `false` — not just missing keys +- String interpolation `\(expr)` only works inside double-quoted jq strings +- `add` on empty array returns `null`, not `0` or `""` — guard with `// []` or `// 0` +- `env.VAR` reads environment variables directly (no `--arg` needed) + +## See Also + +- [jq manual](https://jqlang.github.io/jq/manual/) +- [jqplay.org](https://jqplay.org/) — interactive playground