graph TD
    A[Agent Output] --> B[Stage 1: Deterministic Grader]
    B -->|Pass| C[Stage 2: LLM-as-Judge]
    B -->|Fail| D[Reject + Feedback Loop]
    C -->|Pass| E[Stage 3: Confidence Scoring]
    C -->|Fail| D
    E -->|High Confidence| F[Accept Output]
    E -->|Low Confidence| G{pass@k Available?}
    G -->|Yes| H[Generate k Candidates]
    H --> I[Rank by Grader Consensus]
    I --> F
    G -->|No| J[Human-in-the-Loop Review]
    J --> F
    D --> K[Error Context Injection]
    K --> L[Re-generation with Feedback]
    L --> B

interface Grader {
  name: string;
  type: "deterministic" | "llm_judge" | "human";
  evaluate(output: string, context: VerificationContext): Promise<GradeResult>;
}

interface GradeResult {
  pass: boolean;
  score: number;
  feedback: string;
}

class VerificationPipeline {
  private stages: Grader[][] = [];

  addStage(graders: Grader[]): void {
    this.stages.push(graders);
  }

  async verify(output: string, context: VerificationContext): Promise<VerificationResult> {
    const stageResults: StageResult[] = [];

    for (const [idx, graders] of this.stages.entries()) {
      const grades = await Promise.all(
        graders.map(g => g.evaluate(output, context))
      );
      const stagePassed = grades.every(g => g.pass);
      const stageScore = grades.reduce((sum, g) => sum + g.score, 0) / grades.length;

      stageResults.push({ stage: idx, grades, passed: stagePassed, score: stageScore });

      if (!stagePassed) {
        return {
          accepted: false,
          stageResults,
          feedback: grades.filter(g => !g.pass).map(g => g.feedback).join("\n"),
        };
      }
    }

    const confidence = stageResults.reduce((sum, s) => sum + s.score, 0) / stageResults.length;
    return { accepted: true, stageResults, confidence };
  }
}

async def pass_at_k(task, generator, verifier, k=5):
    """Generate k candidates and select the best by verification score."""
    candidates = await asyncio.gather(*[
        generator.generate(task) for _ in range(k)
    ])

    scored = []
    for candidate in candidates:
        result = await verifier.verify(candidate, task.context)
        scored.append({
            "output": candidate,
            "accepted": result.accepted,
            "confidence": result.confidence,
            "feedback": result.feedback,
        })

    accepted = [s for s in scored if s["accepted"]]
    if accepted:
        return max(accepted, key=lambda s: s["confidence"])

    return max(scored, key=lambda s: s["confidence"])

class LLMJudgeGrader:
    JUDGE_PROMPT = """You are an expert evaluator. Assess the following agent output
against the given task requirements.

Task: {task}
Output: {output}
Criteria: {criteria}

Respond with JSON:
{{"pass": true/false, "score": 0.0-1.0, "feedback": "specific feedback"}}"""

    def __init__(self, judge_model, criteria):
        self.judge_model = judge_model
        self.criteria = criteria

    async def evaluate(self, output, context):
        prompt = self.JUDGE_PROMPT.format(
            task=context.task_description,
            output=output,
            criteria=self.criteria,
        )
        response = await self.judge_model.generate(prompt, temperature=0.0)
        return json.loads(response)

class SchemaGrader:
    def __init__(self, schema: dict):
        self.schema = schema

    async def evaluate(self, output, context):
        try:
            parsed = json.loads(output)
            jsonschema.validate(parsed, self.schema)
            return {"pass": True, "score": 1.0, "feedback": "Schema valid"}
        except jsonschema.ValidationError as e:
            return {"pass": False, "score": 0.0, "feedback": f"Schema violation: {e.message}"}
        except json.JSONDecodeError:
            return {"pass": False, "score": 0.0, "feedback": "Invalid JSON"}