dag-failure-analyzer

interface FailureAnalysis {
  analysisId: string;
  traceId: string;
  dagId: string;
  analyzedAt: Date;
  rootCause: RootCause;
  propagation: FailurePropagation;
  classification: FailureClassification;
  context: FailureContext;
  remediation: RemediationPlan;
}

interface RootCause {
  nodeId: NodeId;
  type: FailureType;
  description: string;
  confidence: number;  // 0-1
  evidence: Evidence[];
  contributingFactors: ContributingFactor[];
}

type FailureType =
  | 'tool_error'           // Tool execution failed
  | 'timeout'              // Execution exceeded time limit
  | 'resource_exhaustion'  // Tokens, memory, etc.
  | 'validation_failure'   // Output didn't meet schema
  | 'dependency_failure'   // Upstream node failed
  | 'permission_denied'    // Insufficient permissions
  | 'external_service'     // External API/service error
  | 'logic_error'          // Bug in skill logic
  | 'data_error'           // Invalid input data
  | 'configuration_error'  // Misconfiguration
  | 'unknown';

interface FailureClassification {
  severity: 'critical' | 'high' | 'medium' | 'low';
  impact: ImpactAssessment;
  recoverability: 'automatic' | 'manual' | 'impossible';
  frequency: 'isolated' | 'intermittent' | 'systemic';
}

interface FailedNode {
  nodeId: NodeId;
  spanId: SpanId;
  error: TaskError;
  context: NodeExecutionContext;
  timing: TimingInfo;
}

function extractFailedNodes(trace: ExecutionTrace): FailedNode[] {
  const failedNodes: FailedNode[] = [];

  for (const [spanId, span] of trace.spans) {
    if (span.status.code === 'ERROR') {
      failedNodes.push({
        nodeId: span.nodeId,
        spanId,
        error: parseError(span.status.message, span.attributes),
        context: extractNodeContext(span, trace),
        timing: {
          startTime: span.startTime,
          endTime: span.endTime,
          duration: span.duration,
        },
      });
    }
  }

  return failedNodes;
}

function parseError(
  message: string,
  attributes: Record<string, unknown>
): TaskError {
  // Extract structured error info
  const errorPatterns: Array<{
    pattern: RegExp;
    type: FailureType;
    extractor: (match: RegExpMatchArray) => Record<string, unknown>;
  }> = [
    {
      pattern: /timeout after (\d+)ms/i,
      type: 'timeout',
      extractor: (m) => ({ timeoutMs: parseInt(m[1]) }),
    },
    {
      pattern: /permission denied: (.+)/i,
      type: 'permission_denied',
      extractor: (m) => ({ deniedResource: m[1] }),
    },
    {
      pattern: /tool "(.+)" failed: (.+)/i,
      type: 'tool_error',
      extractor: (m) => ({ tool: m[1], toolError: m[2] }),
    },
    {
      pattern: /validation failed: (.+)/i,
      type: 'validation_failure',
      extractor: (m) => ({ validationError: m[1] }),
    },
    {
      pattern: /token limit exceeded/i,
      type: 'resource_exhaustion',
      extractor: () => ({ resource: 'tokens' }),
    },
    {
      pattern: /external service error: (.+)/i,
      type: 'external_service',
      extractor: (m) => ({ service: m[1] }),
    },
  ];

  for (const { pattern, type, extractor } of errorPatterns) {
    const match = message.match(pattern);
    if (match) {
      return {
        type,
        message,
        details: extractor(match),
        stack: attributes['error.stack'] as string | undefined,
      };
    }
  }

  return {
    type: 'unknown',
    message,
    details: {},
  };
}

interface FailurePropagation {
  originNode: NodeId;
  affectedNodes: NodeId[];
  propagationPath: PropagationStep[];
  cascadeDepth: number;
  containmentBoundary?: NodeId[];
}

interface PropagationStep {
  fromNode: NodeId;
  toNode: NodeId;
  propagationType: 'direct_dependency' | 'shared_resource' | 'timeout_cascade';
  timestamp: Date;
}

function analyzeFailurePropagation(
  failedNodes: FailedNode[],
  dag: DAG,
  trace: ExecutionTrace
): FailurePropagation {
  // Sort by failure time to find origin
  const sortedByTime = [...failedNodes].sort(
    (a, b) => a.timing.startTime.getTime() - b.timing.startTime.getTime()
  );

  const originNode = sortedByTime[0].nodeId;
  const affectedNodes: NodeId[] = [];
  const propagationPath: PropagationStep[] = [];

  // Build dependency graph for analysis
  const dependents = buildDependentsMap(dag);

  // Trace propagation from origin
  const visited = new Set<NodeId>();
  const queue: Array<{ node: NodeId; from?: NodeId }> = [{ node: originNode }];

  while (queue.length > 0) {
    const current = queue.shift()!;

    if (visited.has(current.node)) continue;
    visited.add(current.node);

    // Check if this node failed
    const failedNode = failedNodes.find(f => f.nodeId === current.node);
    if (failedNode && current.from) {
      affectedNodes.push(current.node);
      propagationPath.push({
        fromNode: current.from,
        toNode: current.node,
        propagationType: determinePropagationType(current.from, current.node, dag),
        timestamp: failedNode.timing.startTime,
      });
    }

    // Add dependents to queue
    const nodeDependent = dependents.get(current.node) ?? [];
    for (const dependent of nodeDependent) {
      queue.push({ node: dependent, from: current.node });
    }
  }

  return {
    originNode,
    affectedNodes,
    propagationPath,
    cascadeDepth: calculateCascadeDepth(propagationPath),
    containmentBoundary: findContainmentBoundary(dag, visited),
  };
}

function buildDependentsMap(dag: DAG): Map<NodeId, NodeId[]> {
  const dependents = new Map<NodeId, NodeId[]>();

  for (const [nodeId, node] of dag.nodes) {
    for (const dep of node.dependencies) {
      const existing = dependents.get(dep) ?? [];
      existing.push(nodeId);
      dependents.set(dep, existing);
    }
  }

  return dependents;
}

function determinePropagationType(
  from: NodeId,
  to: NodeId,
  dag: DAG
): PropagationStep['propagationType'] {
  const toNode = dag.nodes.get(to);
  if (toNode?.dependencies.includes(from)) {
    return 'direct_dependency';
  }
  return 'shared_resource';
}

interface Evidence {
  type: 'error_message' | 'timing' | 'resource_usage' | 'pattern_match';
  source: string;
  observation: string;
  weight: number;  // How strongly this supports the conclusion
}

interface ContributingFactor {
  factor: string;
  contribution: number;  // 0-1
  evidence: Evidence[];
}

function identifyRootCause(
  propagation: FailurePropagation,
  failedNodes: FailedNode[],
  trace: ExecutionTrace,
  history?: FailureHistory
): RootCause {
  const originFailure = failedNodes.find(
    f => f.nodeId === propagation.originNode
  )!;

  const evidence: Evidence[] = [];
  const contributingFactors: ContributingFactor[] = [];

  // Evidence from error message
  evidence.push({
    type: 'error_message',
    source: 'primary_error',
    observation: originFailure.error.message,
    weight: 0.9,
  });

  // Evidence from timing
  if (originFailure.timing.duration && originFailure.timing.duration > 30000) {
    evidence.push({
      type: 'timing',
      source: 'execution_duration',
      observation: `Node ran for ${originFailure.timing.duration}ms before failing`,
      weight: 0.6,
    });
  }

  // Evidence from resource usage
  const resourceUsage = extractResourceUsage(originFailure.context);
  if (resourceUsage.tokensUsed > resourceUsage.tokenLimit * 0.9) {
    evidence.push({
      type: 'resource_usage',
      source: 'token_usage',
      observation: `Used ${resourceUsage.tokensUsed}/${resourceUsage.tokenLimit} tokens (${((resourceUsage.tokensUsed / resourceUsage.tokenLimit) * 100).toFixed(0)}%)`,
      weight: 0.7,
    });
  }

  // Check for pattern matches from history
  if (history) {
    const matchingPatterns = findMatchingPatterns(originFailure, history);
    for (const pattern of matchingPatterns) {
      evidence.push({
        type: 'pattern_match',
        source: 'failure_history',
        observation: `Matches known pattern: ${pattern.name} (seen ${pattern.occurrences} times)`,
        weight: 0.8,
      });
    }
  }

  // Analyze contributing factors
  contributingFactors.push(...analyzeContributingFactors(
    originFailure,
    trace,
    evidence
  ));

  // Calculate confidence based on evidence
  const confidence = calculateConfidence(evidence, contributingFactors);

  return {
    nodeId: propagation.originNode,
    type: originFailure.error.type,
    description: generateRootCauseDescription(originFailure, evidence),
    confidence,
    evidence,
    contributingFactors,
  };
}

function analyzeContributingFactors(
  failure: FailedNode,
  trace: ExecutionTrace,
  evidence: Evidence[]
): ContributingFactor[] {
  const factors: ContributingFactor[] = [];

  // Check for high load (many concurrent nodes)
  const concurrentNodes = countConcurrentNodes(trace, failure.timing.startTime);
  if (concurrentNodes > 5) {
    factors.push({
      factor: 'High concurrent load',
      contribution: Math.min(0.3, concurrentNodes * 0.05),
      evidence: [{
        type: 'timing',
        source: 'concurrency_analysis',
        observation: `${concurrentNodes} nodes executing concurrently`,
        weight: 0.5,
      }],
    });
  }

  // Check for slow dependencies
  const slowDeps = findSlowDependencies(trace, failure.nodeId);
  if (slowDeps.length > 0) {
    factors.push({
      factor: 'Slow upstream dependencies',
      contribution: 0.2,
      evidence: slowDeps.map(dep => ({
        type: 'timing' as const,
        source: 'dependency_analysis',
        observation: `Dependency ${dep.nodeId} took ${dep.duration}ms`,
        weight: 0.4,
      })),
    });
  }

  return factors;
}

function calculateConfidence(
  evidence: Evidence[],
  factors: ContributingFactor[]
): number {
  // Weighted average of evidence weights
  const evidenceTotal = evidence.reduce((sum, e) => sum + e.weight, 0);
  const evidenceAvg = evidenceTotal / Math.max(1, evidence.length);

  // Reduce confidence if many contributing factors
  const factorPenalty = Math.min(0.2, factors.length * 0.05);

  // More evidence = more confidence
  const evidenceBonus = Math.min(0.1, evidence.length * 0.02);

  return Math.max(0.3, Math.min(0.95, evidenceAvg + evidenceBonus - factorPenalty));
}

interface RemediationPlan {
  immediateActions: RemediationAction[];
  preventiveActions: RemediationAction[];
  retryStrategy?: RetryStrategy;
  escalation?: EscalationPlan;
}

interface RemediationAction {
  action: string;
  priority: 'critical' | 'high' | 'medium' | 'low';
  effort: 'trivial' | 'minor' | 'moderate' | 'major';
  expectedImpact: string;
  implementation?: string;
}

interface RetryStrategy {
  recommended: boolean;
  strategy: 'immediate' | 'backoff' | 'skip' | 'manual';
  maxRetries: number;
  backoffMs?: number;
  conditions?: string[];
}

function generateRemediationPlan(
  rootCause: RootCause,
  classification: FailureClassification,
  propagation: FailurePropagation
): RemediationPlan {
  const plan: RemediationPlan = {
    immediateActions: [],
    preventiveActions: [],
  };

  // Generate actions based on failure type
  switch (rootCause.type) {
    case 'timeout':
      plan.immediateActions.push({
        action: 'Increase timeout for affected node',
        priority: 'high',
        effort: 'trivial',
        expectedImpact: 'Allows operation to complete',
        implementation: 'Update node config: timeoutMs: <current * 2>',
      });
      plan.preventiveActions.push({
        action: 'Add progress monitoring to detect slow execution',
        priority: 'medium',
        effort: 'moderate',
        expectedImpact: 'Early warning for timeout-prone operations',
      });
      plan.retryStrategy = {
        recommended: true,
        strategy: 'backoff',
        maxRetries: 2,
        backoffMs: 5000,
        conditions: ['No permanent resource exhaustion'],
      };
      break;

    case 'tool_error':
      plan.immediateActions.push({
        action: 'Check tool availability and permissions',
        priority: 'critical',
        effort: 'trivial',
        expectedImpact: 'Confirms tool is accessible',
      });
      plan.immediateActions.push({
        action: 'Verify tool input parameters',
        priority: 'high',
        effort: 'minor',
        expectedImpact: 'Ensures valid inputs',
      });
      plan.retryStrategy = {
        recommended: false,
        strategy: 'manual',
        maxRetries: 0,
        conditions: ['Tool error must be fixed first'],
      };
      break;

    case 'resource_exhaustion':
      plan.immediateActions.push({
        action: 'Reduce input size or complexity',
        priority: 'high',
        effort: 'moderate',
        expectedImpact: 'Reduces resource requirements',
      });
      plan.preventiveActions.push({
        action: 'Implement chunking for large inputs',
        priority: 'high',
        effort: 'major',
        expectedImpact: 'Prevents future exhaustion',
      });
      plan.retryStrategy = {
        recommended: false,
        strategy: 'skip',
        maxRetries: 0,
        conditions: ['Must reduce resource usage first'],
      };
      break;

    case 'validation_failure':
      plan.immediateActions.push({
        action: 'Review validation rules against actual output',
        priority: 'high',
        effort: 'minor',
        expectedImpact: 'Identifies schema mismatch',
      });
      plan.retryStrategy = {
        recommended: true,
        strategy: 'immediate',
        maxRetries: 2,
        conditions: ['Add validation guidance to prompt'],
      };
      break;

    case 'permission_denied':
      plan.immediateActions.push({
        action: 'Review and update permission matrix',
        priority: 'critical',
        effort: 'minor',
        expectedImpact: 'Grants necessary permissions',
      });
      plan.retryStrategy = {
        recommended: false,
        strategy: 'manual',
        maxRetries: 0,
        conditions: ['Must fix permissions first'],
      };
      break;

    case 'external_service':
      plan.immediateActions.push({
        action: 'Check external service status',
        priority: 'high',
        effort: 'trivial',
        expectedImpact: 'Confirms if service is available',
      });
      plan.retryStrategy = {
        recommended: true,
        strategy: 'backoff',
        maxRetries: 3,
        backoffMs: 10000,
        conditions: ['Service may recover'],
      };
      break;

    default:
      plan.immediateActions.push({
        action: 'Manual investigation required',
        priority: 'high',
        effort: 'moderate',
        expectedImpact: 'Understand failure cause',
      });
  }

  // Add escalation if not recoverable
  if (classification.recoverability === 'impossible') {
    plan.escalation = {
      required: true,
      reason: 'Failure is not automatically recoverable',
      suggestedOwner: 'human',
      context: summarizeForEscalation(rootCause, propagation),
    };
  }

  return plan;
}