crw crawl "https://docs.example.com" -d 2 -l 50         # markdown to stdout
crw crawl "https://docs.example.com/api" -d 1 -l 20 --format json
crw crawl "https://example.com" --js --rate-limit 1.0 --concurrency 3

# Start the crawl
crw_crawl(url="https://docs.example.com", maxDepth=2, maxPages=50)
→ { "id": "a1b2c3d4-..." }

# Poll until status == "completed"
crw_check_crawl_status(id="a1b2c3d4-...")
→ { "status": "scraping|completed|failed", "data": [...] }

# Start
curl -X POST "$CRW_API_URL/v1/crawl" -H "Authorization: Bearer $CRW_API_KEY" \
  -H 'Content-Type: application/json' \
  -d '{"url":"https://docs.example.com","maxDepth":2,"maxPages":50}'
# → {"id":"a1b2c3d4-..."}

# Poll
curl "$CRW_API_URL/v1/crawl/a1b2c3d4-..." \
  -H "Authorization: Bearer $CRW_API_KEY"
# → {"status":"completed","data":[...]}

# Cancel
curl -X DELETE "$CRW_API_URL/v1/crawl/a1b2c3d4-..." \
  -H "Authorization: Bearer $CRW_API_KEY"

loop:
  status = crw_check_crawl_status(id=job_id)
  if status.status == "completed":  break
  if status.status == "failed":     raise error
  wait(3s)

pages = status.data   # list of {url, markdown, html, links, metadata, ...}

crw crawl "https://docs.example.com" -d 2 -l 100 \
  --format json > .crw/crawl-raw.jsonl

# One markdown file per page from the JSON lines
grep '^{' .crw/crawl-raw.jsonl | jq -r '"\(.metadata.sourceURL)\n\(.markdown)"' \
  | split - .crw/pages/page-

# REST: save the full result
curl "$CRW_API_URL/v1/crawl/$JOB_ID" -H "Authorization: Bearer $CRW_API_KEY" \
  | jq -c '.data[]' > .crw/pages.jsonl

# Write one .md per page
jq -r '.markdown' .crw/pages.jsonl | split -l 1 - .crw/pages/page-

1. crw map  "https://docs.example.com" --format json > .crw/urls.json
             → see how many pages exist (check last line: "Discovered N URLs")

2. crw crawl "https://docs.example.com/api" -d 1 -l 20
             → start narrow, verify output quality

3. Scale up: -l 100, -d 2, or scope to a sub-path if needed

4. Write to .crw/, read with grep/jq