Loading...
Loading...
GenAI-first testing with structural assertions and congruence validation. Project-specific test patterns, anti-patterns, and tier structure.
npx skill4agent add akaszubski/autonomous-dev testing-guideassert x == ypytestmark = [pytest.mark.genai]
def test_agents_documented_in_claude_md(self, genai):
agents_on_disk = list_agents()
claude_md = Path("CLAUDE.md").read_text()
result = genai.judge(
question="Does CLAUDE.md document all active agents?",
context=f"Agents on disk: {agents_on_disk}\nCLAUDE.md:\n{claude_md[:3000]}",
criteria="All active agents should be referenced. Score by coverage %."
)
assert result["score"] >= 5, f"Gap: {result['reasoning']}"def test_implement_and_implementer_share_forbidden_list(self, genai):
implement = Path("commands/implement.md").read_text()
implementer = Path("agents/implementer.md").read_text()
result = genai.judge(
question="Do these files have matching FORBIDDEN behavior lists?",
context=f"implement.md:\n{implement[:5000]}\nimplementer.md:\n{implementer[:5000]}",
criteria="Both should define same enforcement gates. Score 10=identical, 0=contradictory."
)
assert result["score"] >= 5def test_all_active_skills_have_content(self):
skills_dir = Path("plugins/autonomous-dev/skills")
for skill in skills_dir.iterdir():
if skill.name == "archived" or not skill.is_dir():
continue
skill_md = skill / "SKILL.md"
assert skill_md.exists(), f"Skill {skill.name} missing SKILL.md"
assert len(skill_md.read_text()) > 100, f"Skill {skill.name} is a hollow shell"# BAD — breaks every time a component is added/removed
assert len(agents) == 14
assert hook_count == 17
# GOOD — minimum thresholds + structural checks
assert len(agents) >= 8, "Pipeline needs at least 8 agents"
assert "implementer.md" in agent_names, "Core agent missing"# BAD — breaks on every config update
assert settings["version"] == "3.51.0"
# GOOD — test structure, not values
assert "version" in settings
assert re.match(r"\d+\.\d+\.\d+", settings["version"])# BAD — breaks on renames/moves
assert Path("plugins/autonomous-dev/lib/old_name.py").exists()
# GOOD — use glob discovery
assert any(Path("plugins/autonomous-dev/lib").glob("*skill*"))@pytest.marktests/
├── regression/
│ ├── smoke/ # Tier 0: Critical path (<5s) — CI GATE
│ ├── regression/ # Tier 1: Feature protection (<30s)
│ ├── extended/ # Tier 2: Deep validation (<5min)
│ └── progression/ # Tier 3: TDD red phase (not yet implemented)
├── unit/ # Isolated functions (<1s each)
├── integration/ # Multi-component workflows (<30s)
├── genai/ # LLM-as-judge (opt-in via --genai flag)
└── archived/ # Excluded from runsregression/smoke/regression/regression/unit/integration/genai/pytest -m smoke # CI gate
pytest -m "smoke or regression" # Feature protection
pytest tests/genai/ --genai # GenAI validation (opt-in)# tests/genai/conftest.py provides two fixtures:
# - genai: Gemini Flash via OpenRouter (cheap, fast)
# - genai_smart: Haiku 4.5 via OpenRouter (complex reasoning)
# Requires: OPENROUTER_API_KEY env var + --genai pytest flag
# Cost: ~$0.02 per full run with 24h response caching/scaffold-genai-uattests/genai/| Test This | With This | Not This |
|---|---|---|
| Pure Python functions | Unit tests | — |
| Component interactions | Integration tests | — |
| Doc ↔ code alignment | GenAI congruence | Hardcoded string matching |
| Component existence | Structural (glob) | Hardcoded counts |
| FORBIDDEN list sync | GenAI congruence | Manual comparison |
| Security posture | GenAI judge | Regex scanning |
| Config structure | Structural | Config values |
| Agent output quality | GenAI judge | Output string matching |
test_regression_issue_NNN_description--genai