docs: add curl howto
Covers HTTP methods, headers, auth, file upload/download, cookies, response inspection with -w, proxies, and practical recipes.
This commit is contained in:
user
2
TODO.md
2
TODO.md
@@ -6,7 +6,7 @@
|
||||
- [x] ansible — playbook patterns, inventory, vault, variables, roles
|
||||
- [ ] podman — build, run, compose, volumes
|
||||
- [x] jq — filters, select, map, slurp, recipes, @format
|
||||
- [ ] curl — headers, auth, methods, output
|
||||
- [x] curl — methods, headers, auth, upload, download, cookies, -w format
|
||||
- [ ] systemd — units, journalctl, timers
|
||||
- [ ] ssh — config, tunnels, keys, agent
|
||||
- [ ] bash — parameter expansion, arrays, traps
|
||||
|
||||
318
topics/curl.md
Normal file
318
topics/curl.md
Normal file
@@ -0,0 +1,318 @@
|
||||
# curl
|
||||
|
||||
> Transfer data to/from servers — the Swiss army knife of HTTP.
|
||||
|
||||
## Basics
|
||||
|
||||
```bash
|
||||
# Simple GET
|
||||
curl https://example.com
|
||||
|
||||
# Follow redirects
|
||||
curl -L https://example.com/old-path
|
||||
|
||||
# Silent (no progress bar)
|
||||
curl -s https://api.example.com/data
|
||||
|
||||
# Silent but show errors
|
||||
curl -sS https://api.example.com/data
|
||||
|
||||
# Save to file
|
||||
curl -o output.html https://example.com
|
||||
curl -O https://example.com/file.tar.gz # keep remote filename
|
||||
|
||||
# Show response headers + body
|
||||
curl -i https://example.com
|
||||
|
||||
# Show headers only
|
||||
curl -I https://example.com # HEAD request
|
||||
curl -sI https://example.com # silent + headers only
|
||||
|
||||
# Verbose (debug connection, TLS, headers)
|
||||
curl -v https://example.com
|
||||
curl -vvv https://example.com # extra verbose
|
||||
```
|
||||
|
||||
## Common Flags
|
||||
|
||||
| Flag | Effect |
|
||||
|--------------------|-----------------------------------------------|
|
||||
| `-s` | Silent (no progress meter) |
|
||||
| `-S` | Show errors even when silent |
|
||||
| `-L` | Follow redirects |
|
||||
| `-i` | Include response headers in output |
|
||||
| `-I` | Headers only (HEAD request) |
|
||||
| `-v` | Verbose (show request/response details) |
|
||||
| `-o FILE` | Write output to file |
|
||||
| `-O` | Save with remote filename |
|
||||
| `-X METHOD` | HTTP method (GET, POST, PUT, DELETE, PATCH) |
|
||||
| `-H "K: V"` | Add request header |
|
||||
| `-d DATA` | POST data (sets method to POST) |
|
||||
| `-F "k=v"` | Multipart form data (file upload) |
|
||||
| `-u user:pass` | Basic auth |
|
||||
| `-b FILE` | Send cookies from file |
|
||||
| `-c FILE` | Save cookies to file |
|
||||
| `-k` | Ignore TLS certificate errors |
|
||||
| `-w FORMAT` | Write-out format string after transfer |
|
||||
| `--connect-timeout N` | Connection timeout in seconds |
|
||||
| `-m N` | Max total time in seconds |
|
||||
| `--retry N` | Retry N times on transient errors |
|
||||
| `--compressed` | Request and decompress gzip/deflate/br |
|
||||
|
||||
## HTTP Methods
|
||||
|
||||
```bash
|
||||
# GET (default)
|
||||
curl https://api.example.com/users
|
||||
|
||||
# POST — form-encoded (default content-type with -d)
|
||||
curl -X POST -d "name=alice&role=admin" https://api.example.com/users
|
||||
|
||||
# POST — JSON
|
||||
curl -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "alice", "role": "admin"}' \
|
||||
https://api.example.com/users
|
||||
|
||||
# POST — JSON from file
|
||||
curl -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @payload.json \
|
||||
https://api.example.com/users
|
||||
|
||||
# POST — read body from stdin
|
||||
echo '{"key":"value"}' | curl -X POST -H "Content-Type: application/json" -d @- https://api.example.com
|
||||
|
||||
# PUT
|
||||
curl -X PUT \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"role": "superadmin"}' \
|
||||
https://api.example.com/users/42
|
||||
|
||||
# PATCH
|
||||
curl -X PATCH \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"role": "viewer"}' \
|
||||
https://api.example.com/users/42
|
||||
|
||||
# DELETE
|
||||
curl -X DELETE https://api.example.com/users/42
|
||||
```
|
||||
|
||||
## Headers
|
||||
|
||||
```bash
|
||||
# Set custom headers
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
-H "Accept: application/json" \
|
||||
-H "X-Request-ID: abc-123" \
|
||||
https://api.example.com/data
|
||||
|
||||
# Content-Type shortcuts
|
||||
curl -H "Content-Type: application/json" -d '...'
|
||||
curl -H "Content-Type: text/xml" -d @request.xml
|
||||
|
||||
# User-Agent
|
||||
curl -A "my-script/1.0" https://example.com
|
||||
|
||||
# Referer
|
||||
curl -e "https://example.com/origin" https://example.com/target
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
```bash
|
||||
# Basic auth
|
||||
curl -u alice:s3cret https://api.example.com/private
|
||||
|
||||
# Basic auth (prompt for password)
|
||||
curl -u alice https://api.example.com/private
|
||||
|
||||
# Bearer token
|
||||
curl -H "Authorization: Bearer $TOKEN" https://api.example.com
|
||||
|
||||
# API key in header
|
||||
curl -H "X-API-Key: $API_KEY" https://api.example.com
|
||||
|
||||
# Netrc file (~/.netrc)
|
||||
curl -n https://api.example.com
|
||||
# ~/.netrc format:
|
||||
# machine api.example.com login alice password s3cret
|
||||
|
||||
# Client certificate (mTLS)
|
||||
curl --cert client.pem --key client-key.pem https://secure.example.com
|
||||
```
|
||||
|
||||
## File Upload
|
||||
|
||||
```bash
|
||||
# Multipart form upload
|
||||
curl -F "file=@document.pdf" https://example.com/upload
|
||||
curl -F "file=@photo.jpg;type=image/jpeg" https://example.com/upload
|
||||
|
||||
# Multiple files
|
||||
curl -F "file1=@doc.pdf" -F "file2=@img.png" https://example.com/upload
|
||||
|
||||
# File + form fields
|
||||
curl -F "file=@doc.pdf" -F "title=Report" -F "public=true" https://example.com/upload
|
||||
|
||||
# PUT upload (raw file body)
|
||||
curl -T file.tar.gz https://example.com/upload/file.tar.gz
|
||||
```
|
||||
|
||||
## Download
|
||||
|
||||
```bash
|
||||
# Save with remote filename
|
||||
curl -O https://example.com/archive.tar.gz
|
||||
|
||||
# Save to specific path
|
||||
curl -o /tmp/data.json https://api.example.com/export
|
||||
|
||||
# Resume interrupted download
|
||||
curl -C - -O https://example.com/large-file.iso
|
||||
|
||||
# Download multiple files
|
||||
curl -O https://example.com/file1.txt -O https://example.com/file2.txt
|
||||
|
||||
# Rate limit
|
||||
curl --limit-rate 1M -O https://example.com/large.iso
|
||||
|
||||
# Show progress bar (instead of meter)
|
||||
curl -# -O https://example.com/file.tar.gz
|
||||
```
|
||||
|
||||
## Cookies
|
||||
|
||||
```bash
|
||||
# Send cookie
|
||||
curl -b "session=abc123" https://example.com
|
||||
|
||||
# Save cookies from response
|
||||
curl -c cookies.txt https://example.com/login
|
||||
|
||||
# Load and send saved cookies
|
||||
curl -b cookies.txt https://example.com/dashboard
|
||||
|
||||
# Save + send (session flow)
|
||||
curl -c cookies.txt -b cookies.txt \
|
||||
-d "user=alice&pass=secret" \
|
||||
https://example.com/login
|
||||
```
|
||||
|
||||
## Response Inspection with -w
|
||||
|
||||
```bash
|
||||
# HTTP status code only
|
||||
curl -s -o /dev/null -w "%{http_code}" https://example.com
|
||||
|
||||
# Timing breakdown
|
||||
curl -s -o /dev/null -w "\
|
||||
dns: %{time_namelookup}s\n\
|
||||
connect: %{time_connect}s\n\
|
||||
tls: %{time_appconnect}s\n\
|
||||
start: %{time_starttransfer}s\n\
|
||||
total: %{time_total}s\n\
|
||||
size: %{size_download} bytes\n\
|
||||
code: %{http_code}\n" \
|
||||
https://example.com
|
||||
|
||||
# Response content-type
|
||||
curl -s -o /dev/null -w "%{content_type}" https://example.com
|
||||
|
||||
# Effective URL after redirects
|
||||
curl -sL -o /dev/null -w "%{url_effective}" https://short.url/abc
|
||||
|
||||
# Write-out to file
|
||||
curl -s -o /dev/null -w "%{http_code}" -o response.json https://api.example.com
|
||||
```
|
||||
|
||||
### Useful -w Variables
|
||||
|
||||
| Variable | Value |
|
||||
|-------------------------|------------------------------------|
|
||||
| `%{http_code}` | HTTP status code |
|
||||
| `%{time_total}` | Total time in seconds |
|
||||
| `%{time_namelookup}` | DNS resolution time |
|
||||
| `%{time_connect}` | TCP connection time |
|
||||
| `%{time_appconnect}` | TLS handshake time |
|
||||
| `%{time_starttransfer}` | Time to first byte (TTFB) |
|
||||
| `%{size_download}` | Downloaded bytes |
|
||||
| `%{size_upload}` | Uploaded bytes |
|
||||
| `%{url_effective}` | Final URL after redirects |
|
||||
| `%{content_type}` | Content-Type header |
|
||||
| `%{num_redirects}` | Number of redirects followed |
|
||||
| `%{remote_ip}` | Server IP address |
|
||||
|
||||
## Proxies
|
||||
|
||||
```bash
|
||||
# HTTP proxy
|
||||
curl -x http://proxy:8080 https://example.com
|
||||
|
||||
# SOCKS5 proxy
|
||||
curl --socks5-hostname localhost:1080 https://example.com
|
||||
|
||||
# No proxy for specific hosts
|
||||
curl --noproxy "localhost,*.internal" https://example.com
|
||||
|
||||
# Env variable (respected by curl)
|
||||
export https_proxy=http://proxy:8080
|
||||
curl https://example.com
|
||||
```
|
||||
|
||||
## Practical Recipes
|
||||
|
||||
```bash
|
||||
# JSON API call with jq
|
||||
curl -s https://api.example.com/users | jq '.[] | {name, email}'
|
||||
|
||||
# Check if endpoint is reachable
|
||||
curl -sf -o /dev/null https://example.com && echo "up" || echo "down"
|
||||
|
||||
# POST JSON, extract field from response
|
||||
ID=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name":"test"}' \
|
||||
https://api.example.com/items | jq -r '.id')
|
||||
|
||||
# Retry with backoff
|
||||
curl --retry 5 --retry-delay 2 --retry-all-errors https://api.example.com
|
||||
|
||||
# Send from config file (avoid long command lines)
|
||||
curl -K request.conf
|
||||
# request.conf:
|
||||
# url = "https://api.example.com"
|
||||
# header = "Authorization: Bearer token123"
|
||||
# header = "Accept: application/json"
|
||||
# silent
|
||||
|
||||
# Loop over paginated API
|
||||
PAGE=1; while :; do
|
||||
DATA=$(curl -s "https://api.example.com/items?page=$PAGE&limit=100")
|
||||
echo "$DATA" | jq '.items[]'
|
||||
[ "$(echo "$DATA" | jq '.items | length')" -lt 100 ] && break
|
||||
((PAGE++))
|
||||
done
|
||||
|
||||
# Parallel downloads with xargs
|
||||
cat urls.txt | xargs -P 4 -I{} curl -sO {}
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- `-d` implies POST and sets `Content-Type: application/x-www-form-urlencoded` — add `-H` for JSON
|
||||
- `-X GET -d '...'` sends a body with GET — legal but unusual; some servers reject it
|
||||
- Without `-L`, 3xx redirects return the redirect response, not the final destination
|
||||
- `-k` disables all TLS verification — never use in production scripts
|
||||
- `-o /dev/null` is needed with `-w` to suppress body when you only want status/timing
|
||||
- Spaces in URLs must be encoded (`%20`) or the URL must be quoted
|
||||
- `--data-urlencode` handles encoding for form values; plain `-d` does not
|
||||
- Shell variables in single-quoted `-d '...'` won't expand — use double quotes or `--data-raw`
|
||||
- `-F` and `-d` cannot be mixed in the same request
|
||||
|
||||
## See Also
|
||||
|
||||
- `jq` — pipe curl JSON output through jq for parsing
|
||||
- [Everything curl](https://everything.curl.dev/) — comprehensive book
|
||||
- `curl --help all` — full flag reference
|
||||
Reference in New Issue
Block a user