crw search "query" --json --limit 5
crw scrape "https://example.com" --format json
crw map "https://docs.example.com"
crw scrape "report.pdf"                              # local PDF auto-detected

crw_scrape(url="https://example.com", formats=["markdown"], onlyMainContent=true)
crw_search(query="query", limit=5)
crw_map(url="https://docs.example.com", limit=200)

undefined

**Use REST when:** writing application code, needing async crawl jobs with polling,
or integrating with frameworks like LangChain / CrewAI / LlamaIndex.

undefined

undefined

keywords = {'commercializ', 'battery', 'production', '2025', '2026'}
relevant = [r for r in results
            if any(kw in r['description'].lower() for kw in keywords)]

for para in markdown.split('\n\n'):
    if len(para) > 60 and any(kw in para.lower() for kw in keywords):
        print(para)

import anthropic

def is_relevant(snippet: str, query: str) -> dict:
    """Returns {is_match: bool, confidence: float, reasoning: str}"""
    client = anthropic.Anthropic()
    msg = client.messages.create(
        model="claude-haiku-4-5",   # cheap model for classification
        max_tokens=128,
        messages=[{
            "role": "user",
            "content": (
                f"Query: {query}\n\n"
                f"Snippet: {snippet[:500]}\n\n"
                "Does this snippet directly answer or provide evidence for the query? "
                "Reply with JSON only: {\"is_match\": true/false, \"confidence\": 0-1, "
                "\"reasoning\": \"one sentence\"}"
            )
        }]
    )
    import json
    return json.loads(msg.content[0].text)

crw search "query" → top-N results (titles + snippets)
→ scrape top 3-5 full pages → filter to relevant paragraphs
→ embed filtered paragraphs → merge with local vector store
→ retrieve top-K chunks → feed to generation model

from crw import CrwClient

client = CrwClient()  # embedded mode, no server

def retrieve_and_chunk(query: str, top_n: int = 5) -> list[str]:
    results = client.search(query, limit=top_n)
    chunks = []
    for r in results:
        # Scrape full page if the snippet isn't sufficient
        page = client.scrape(r['url'], formats=['markdown'])
        md = page.get('markdown', '') or ''
        # Split into paragraphs, keep non-trivial ones
        for para in md.split('\n\n'):
            para = para.strip()
            if len(para) > 100:
                chunks.append(para)
    return chunks