COLLECT → ENRICH → STORE
  │           │        │
Scraper    AI (LLM)  Database
runs on    scores/   Notion /
schedule   summarises Sheets /
           & classifies Supabase

gemini-2.0-flash-lite (30 RPM) →
gemini-2.0-flash (15 RPM) →
gemini-2.5-flash (10 RPM) →
gemini-flash-lite-latest (fallback)

undefined

---

my-agent/
├── config.yaml              # User customises this (keywords, filters, preferences)
├── profile/
│   └── context.md           # User context the AI uses (resume, interests, criteria)
├── scraper/
│   ├── __init__.py
│   ├── main.py              # Orchestrator: scrape → enrich → store
│   ├── filters.py           # Rule-based pre-filter (fast, before AI)
│   └── sources/
│       ├── __init__.py
│       └── source_name.py   # One file per data source
├── ai/
│   ├── __init__.py
│   ├── client.py            # Gemini REST client with model fallback
│   ├── pipeline.py          # Batch AI analysis
│   ├── jd_fetcher.py        # Fetch full content from URLs (optional)
│   └── memory.py            # Learn from user feedback
├── storage/
│   ├── __init__.py
│   └── notion_sync.py       # Or sheets_sync.py / supabase_sync.py
├── data/
│   └── feedback.json        # User decision history (auto-updated)
├── .env.example
├── setup.py                 # One-time DB/schema creation
├── enrich_existing.py       # Backfill AI scores on old rows
├── requirements.txt
└── .github/
    └── workflows/
        └── scraper.yml      # GitHub Actions schedule

undefined

# ---- REST API source ----
resp = requests.get("https://api.example.com/items", headers=HEADERS, timeout=15)
if resp.status_code == 200:
    for item in resp.json().get("results", []):
        if not is_relevant(item.get("title", "")):
            continue
        results.append(_normalise(item))

return results

**HTML scraping pattern:**
```python
soup = BeautifulSoup(resp.text, "lxml")
for card in soup.select("[class*='listing']"):
    title = card.select_one("h2, h3").get_text(strip=True)
    link = card.select_one("a")["href"]
    if not link.startswith("http"):
        link = f"https://example.com{link}"

import xml.etree.ElementTree as ET
root = ET.fromstring(resp.text)
for item in root.findall(".//item"):
    title = item.findtext("title", "")
    link = item.findtext("link", "")

undefined

api_key = os.environ.get("GEMINI_API_KEY", "")
if not api_key:
    return {}

elapsed = time.time() - _last_call
if elapsed < rate_limit:
    time.sleep(rate_limit - elapsed)

models = [model] + [m for m in MODEL_FALLBACK if m != model] if model else MODEL_FALLBACK
_last_call = time.time()

for m in models:
    url = f"https://generativelanguage.googleapis.com/v1beta/models/{m}:generateContent?key={api_key}"
    payload = {
        "contents": [{"parts": [{"text": prompt}]}],
        "generationConfig": {
            "responseMimeType": "application/json",
            "temperature": 0.3,
            "maxOutputTokens": 2048,
        },
    }
    try:
        resp = requests.post(url, json=payload, timeout=30)
        if resp.status_code == 200:
            return _parse(resp)
        if resp.status_code in (429, 404):
            time.sleep(1)
            continue
        return {}
    except requests.RequestException:
        return {}

return {}

---

undefined

batches = [items[i:i + batch_size] for i in range(0, len(items), batch_size)]
print(f"  [AI] {len(items)} items → {len(batches)} API calls")

enriched = []
for i, batch in enumerate(batches):
    print(f"  [AI] Batch {i + 1}/{len(batches)}...")
    prompt = _build_prompt(batch, context, preference_prompt, config)
    result = generate(prompt, model=model, rate_limit=rate_limit)

    analyses = result.get("analyses", [])
    for j, item in enumerate(batch):
        ai = analyses[j] if j < len(analyses) else {}
        if ai:
            score = max(0, min(100, int(ai.get("score", 0))))
            if min_score and score < min_score:
                continue
            enriched.append({**item, "ai_score": score, "ai_summary": ai.get("summary", ""), "ai_notes": ai.get("notes", "")})
        else:
            enriched.append(item)

return enriched

return f"""Analyse these {len(batch)} items and return a JSON object.

---

undefined

**Integration with your storage layer:** after each run, query your DB for items with positive/negative status and call `save_feedback()` with the extracted patterns.

---

undefined

try:
    get_client().pages.create(parent={"database_id": db_id}, properties=props)
    return True
except APIResponseError as e:
    print(f"[notion] Push failed: {e}")
    return False

---

undefined

# Resolve the storage target identifier from env based on provider
if provider == "notion":
    db_id = os.environ.get("NOTION_DATABASE_ID")
    if not db_id:
        print("ERROR: NOTION_DATABASE_ID not set"); sys.exit(1)
else:
    # Extend here for sheets (SHEET_ID) or supabase (SUPABASE_TABLE) etc.
    print(f"ERROR: provider '{provider}' not yet wired in main.py"); sys.exit(1)

config = yaml.safe_load((Path(__file__).parent.parent / "config.yaml").read_text())
all_items = []

for name, fetch_fn in SOURCES:
    try:
        items = fetch_fn()
        print(f"[{name}] {len(items)} items")
        all_items.extend(items)
    except Exception as e:
        print(f"[{name}] FAILED: {e}")

# Deduplicate by URL
seen, deduped = set(), []
for item in all_items:
    if (url := item.get("url", "")) and url not in seen:
        seen.add(url); deduped.append(item)

print(f"Unique items: {len(deduped)}")

if ai_enabled() and deduped:
    from ai.memory import load_feedback, build_preference_prompt
    from ai.pipeline import analyse_batch

    # load_feedback() reads data/feedback.json written by your feedback sync script.
    # To keep it current, implement a separate feedback_sync.py that queries your
    # storage provider for items with positive/negative statuses and calls save_feedback().
    feedback = load_feedback()
    preference = build_preference_prompt(feedback)
    context_path = Path(__file__).parent.parent / "profile" / "context.md"
    context = context_path.read_text() if context_path.exists() else ""
    deduped = analyse_batch(deduped, context=context, preference_prompt=preference)
else:
    print("[AI] Skipped — GEMINI_API_KEY not set")

added, skipped = sync(db_id, deduped)
print(f"Done — {added} new, {skipped} existing")

---

undefined

steps:
  - uses: actions/checkout@v4

  - uses: actions/setup-python@v5
    with:
      python-version: "3.11"
      cache: "pip"

  - run: pip install -r requirements.txt

  # Uncomment if Playwright is enabled in requirements.txt
  # - name: Install Playwright browsers
  #   run: python -m playwright install chromium --with-deps

  - name: Run agent
    env:
      NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
      NOTION_DATABASE_ID: ${{ secrets.NOTION_DATABASE_ID }}
      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
    run: python -m scraper.main

  - name: Commit feedback history
    run: |
      git config user.name "github-actions[bot]"
      git config user.email "github-actions[bot]@users.noreply.github.com"
      git add data/feedback.json || true
      git diff --cached --quiet || git commit -m "chore: update feedback history"
      git push

---

undefined

---

resp = requests.get(url, params={"q": query}, headers=HEADERS, timeout=15)
items = resp.json().get("results", [])

soup = BeautifulSoup(resp.text, "lxml")
for card in soup.select(".listing-card"):
    title = card.select_one("h2").get_text(strip=True)
    href = card.select_one("a")["href"]

import xml.etree.ElementTree as ET
root = ET.fromstring(resp.text)
for item in root.findall(".//item"):
    title = item.findtext("title", "")
    link = item.findtext("link", "")
    pub_date = item.findtext("pubDate", "")

page = 1
while True:
    resp = requests.get(url, params={"page": page, "limit": 50}, timeout=15)
    data = resp.json()
    items = data.get("results", [])
    if not items:
        break
    for item in items:
        results.append(_normalise(item))
    if not data.get("has_more"):
        break
    page += 1

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto(url)
    page.wait_for_selector(".listing")
    html = page.content()
    browser.close()

soup = BeautifulSoup(html, "lxml")

requests==2.31.0
beautifulsoup4==4.12.3
lxml==5.1.0
python-dotenv==1.0.1
pyyaml==6.0.2
notion-client==2.2.1   # if using Notion

---

"Build me an agent that monitors Hacker News for AI startup funding news"
"Scrape product prices from 3 e-commerce sites and alert when they drop"
"Track new GitHub repos tagged with 'llm' or 'agents' — summarise each one"
"Collect Chief of Staff job listings from LinkedIn and Cutshort into Notion"
"Monitor a subreddit for posts mentioning my company — classify sentiment"
"Scrape new academic papers from arXiv on a topic I care about daily"
"Track sports fixture results and keep a running table in Google Sheets"
"Build a real estate listing watcher — alert on new properties under ₹1 Cr"