pipeline_name/
  README.md
  data-pipeline-spec.md
  config/
    pipeline_config.yaml
    connections.yaml.example
    .env.example
  src/
    extractors/
      __init__.py
      base_extractor.py
      [source_name]_extractor.py
    transformers/
      __init__.py
      base_transformer.py
      [transform_name]_transformer.py
    loaders/
      __init__.py
      base_loader.py
      [destination_name]_loader.py
    quality/
      __init__.py
      checks.py
      expectations.py
    utils/
      __init__.py
      logging_config.py
      retry.py
      metrics.py
  sql/
    staging/
      [table_name]_staging.sql
    transformations/
      [transform_name].sql
    quality_checks/
      [check_name].sql
  orchestration/
    dags/
      [pipeline_name]_dag.py
    schedules/
      schedule_config.yaml
  tests/
    unit/
      test_extractors.py
      test_transformers.py
      test_loaders.py
      test_quality.py
    integration/
      test_pipeline_e2e.py
    fixtures/
      sample_data/
  monitoring/
    alerts/
      alert_rules.yaml
    dashboards/
      pipeline_dashboard.json
  docker/
    Dockerfile
    docker-compose.yaml
  requirements.txt
  pyproject.toml
  Makefile

"""Base extractor module providing the abstract interface for all data extractors."""

from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Generator, Optional

import structlog

logger = structlog.get_logger(__name__)


@dataclass
class ExtractionResult:
    """Container for extraction results with metadata."""

    data: list[dict[str, Any]]
    source_name: str
    extraction_timestamp: datetime
    record_count: int
    watermark_value: Optional[str] = None
    schema_hash: Optional[str] = None

    def __post_init__(self) -> None:
        if self.record_count != len(self.data):
            raise ValueError(
                f"Record count mismatch: declared {self.record_count}, "
                f"actual {len(self.data)}"
            )


class BaseExtractor(ABC):
    """Abstract base class for all data extractors.

    Subclasses must implement `connect`, `extract`, and `close` methods.
    The base class provides retry logic, logging, and watermark tracking.
    """

    def __init__(self, config: dict[str, Any], source_name: str) -> None:
        self.config = config
        self.source_name = source_name
        self._connected = False
        self._log = logger.bind(source=source_name)

    @abstractmethod
    def connect(self) -> None:
        """Establish connection to the data source."""
        ...

    @abstractmethod
    def extract(
        self,
        watermark: Optional[str] = None,
        batch_size: int = 10000,
    ) -> Generator[ExtractionResult, None, None]:
        """Extract data from the source, yielding batches.

        Args:
            watermark: Resume point for incremental extraction.
            batch_size: Number of records per batch.

        Yields:
            ExtractionResult for each batch of extracted data.
        """
        ...

    @abstractmethod
    def close(self) -> None:
        """Clean up connections and resources."""
        ...

    def __enter__(self) -> "BaseExtractor":
        self.connect()
        self._connected = True
        return self

    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
        self.close()
        self._connected = False

    def validate_connection(self) -> bool:
        """Test that the source connection is alive and responsive."""
        try:
            self.connect()
            self.close()
            return True
        except Exception as exc:
            self._log.error("connection_validation_failed", error=str(exc))
            return False

"""Base transformer module providing the abstract interface for all transformations."""

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Optional

import structlog

logger = structlog.get_logger(__name__)


@dataclass
class TransformationResult:
    """Container for transformation results with lineage metadata."""

    data: list[dict[str, Any]]
    transform_name: str
    input_record_count: int
    output_record_count: int
    transformation_timestamp: datetime
    dropped_records: int = 0
    error_records: list[dict[str, Any]] = field(default_factory=list)
    lineage: Optional[dict[str, Any]] = None

    @property
    def drop_rate(self) -> float:
        """Calculate the percentage of records dropped during transformation."""
        if self.input_record_count == 0:
            return 0.0
        return self.dropped_records / self.input_record_count * 100


class BaseTransformer(ABC):
    """Abstract base class for all data transformers.

    Subclasses must implement `transform` and `validate_output`.
    Provides standard logging, error collection, and lineage tracking.
    """

    def __init__(self, config: dict[str, Any], transform_name: str) -> None:
        self.config = config
        self.transform_name = transform_name
        self._log = logger.bind(transform=transform_name)

    @abstractmethod
    def transform(
        self,
        data: list[dict[str, Any]],
    ) -> TransformationResult:
        """Apply transformation logic to the input data.

        Args:
            data: List of records to transform.

        Returns:
            TransformationResult containing transformed data and metadata.
        """
        ...

    @abstractmethod
    def validate_output(
        self,
        result: TransformationResult,
    ) -> list[str]:
        """Validate the transformation output against business rules.

        Args:
            result: The transformation result to validate.

        Returns:
            List of validation error messages. Empty list means valid.
        """
        ...

    def _track_lineage(
        self,
        input_sources: list[str],
        output_fields: list[str],
        logic_description: str,
    ) -> dict[str, Any]:
        """Generate lineage metadata for the transformation."""
        return {
            "transform_name": self.transform_name,
            "input_sources": input_sources,
            "output_fields": output_fields,
            "logic": logic_description,
            "timestamp": datetime.utcnow().isoformat(),
        }

"""Base loader module providing the abstract interface for all data loaders."""

from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, Optional

import structlog

logger = structlog.get_logger(__name__)


class LoadStrategy(Enum):
    """Supported load strategies."""

    APPEND = "append"
    OVERWRITE = "overwrite"
    UPSERT = "upsert"
    MERGE = "merge"
    SCD_TYPE_2 = "scd_type_2"


@dataclass
class LoadResult:
    """Container for load operation results."""

    destination_name: str
    strategy: LoadStrategy
    records_loaded: int
    records_updated: int
    records_rejected: int
    load_timestamp: datetime
    duration_seconds: float
    partition_key: Optional[str] = None
    rejected_records: list[dict[str, Any]] = None

    def __post_init__(self) -> None:
        if self.rejected_records is None:
            self.rejected_records = []

    @property
    def success_rate(self) -> float:
        """Calculate the percentage of records successfully loaded."""
        total = self.records_loaded + self.records_rejected
        if total == 0:
            return 100.0
        return self.records_loaded / total * 100


class BaseLoader(ABC):
    """Abstract base class for all data loaders.

    Subclasses must implement `connect`, `load`, and `close` methods.
    The base class provides transaction management, retry logic, and metrics.
    """

    def __init__(
        self,
        config: dict[str, Any],
        destination_name: str,
        strategy: LoadStrategy = LoadStrategy.APPEND,
    ) -> None:
        self.config = config
        self.destination_name = destination_name
        self.strategy = strategy
        self._connected = False
        self._log = logger.bind(destination=destination_name, strategy=strategy.value)

    @abstractmethod
    def connect(self) -> None:
        """Establish connection to the destination system."""
        ...

    @abstractmethod
    def load(
        self,
        data: list[dict[str, Any]],
        target_table: str,
    ) -> LoadResult:
        """Load data into the destination.

        Args:
            data: List of records to load.
            target_table: Target table or collection name.

        Returns:
            LoadResult with load operation metadata.
        """
        ...

    @abstractmethod
    def close(self) -> None:
        """Clean up connections and resources."""
        ...

    def __enter__(self) -> "BaseLoader":
        self.connect()
        self._connected = True
        return self

    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
        self.close()
        self._connected = False

    @abstractmethod
    def create_table_if_not_exists(
        self,
        table_name: str,
        schema: dict[str, str],
    ) -> None:
        """Ensure the target table exists with the correct schema.

        Args:
            table_name: Name of the table to create.
            schema: Column name to data type mapping.
        """
        ...

"""Data quality check module for validating pipeline data at every stage."""

from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from typing import Any, Callable, Optional

import structlog

logger = structlog.get_logger(__name__)


class CheckSeverity(Enum):
    """Severity level for quality check failures."""

    WARN = "warn"
    ERROR = "error"
    CRITICAL = "critical"


class CheckStatus(Enum):
    """Result status of a quality check."""

    PASSED = "passed"
    FAILED = "failed"
    SKIPPED = "skipped"


@dataclass
class QualityCheckResult:
    """Result of a single data quality check."""

    check_name: str
    status: CheckStatus
    severity: CheckSeverity
    message: str
    checked_at: datetime
    records_checked: int
    records_failed: int = 0
    failed_examples: list[dict[str, Any]] = field(default_factory=list)
    metadata: Optional[dict[str, Any]] = None

    @property
    def failure_rate(self) -> float:
        """Calculate the percentage of records that failed the check."""
        if self.records_checked == 0:
            return 0.0
        return self.records_failed / self.records_checked * 100


@dataclass
class QualitySuiteResult:
    """Aggregate result of running a full quality check suite."""

    suite_name: str
    results: list[QualityCheckResult]
    executed_at: datetime
    duration_seconds: float

    @property
    def passed(self) -> bool:
        """Return True if no ERROR or CRITICAL checks failed."""
        return not any(
            r.status == CheckStatus.FAILED
            and r.severity in (CheckSeverity.ERROR, CheckSeverity.CRITICAL)
            for r in self.results
        )

    @property
    def summary(self) -> dict[str, int]:
        """Return count of checks by status."""
        counts: dict[str, int] = {"passed": 0, "failed": 0, "skipped": 0}
        for r in self.results:
            counts[r.status.value] += 1
        return counts


class QualityCheck:
    """A single configurable data quality check.

    Checks are composable and can be combined into suites.
    Each check is a function that takes data and returns a boolean.
    """

    def __init__(
        self,
        name: str,
        description: str,
        check_fn: Callable[[list[dict[str, Any]]], tuple[bool, list[dict[str, Any]]]],
        severity: CheckSeverity = CheckSeverity.ERROR,
    ) -> None:
        self.name = name
        self.description = description
        self.check_fn = check_fn
        self.severity = severity

    def run(self, data: list[dict[str, Any]]) -> QualityCheckResult:
        """Execute the quality check against the provided data.

        Args:
            data: List of records to check.

        Returns:
            QualityCheckResult with pass/fail status and details.
        """
        log = logger.bind(check=self.name)
        try:
            passed, failed_records = self.check_fn(data)
            status = CheckStatus.PASSED if passed else CheckStatus.FAILED
            log.info("check_completed", status=status.value, failures=len(failed_records))
            return QualityCheckResult(
                check_name=self.name,
                status=status,
                severity=self.severity,
                message=self.description,
                checked_at=datetime.utcnow(),
                records_checked=len(data),
                records_failed=len(failed_records),
                failed_examples=failed_records[:10],
            )
        except Exception as exc:
            log.error("check_error", error=str(exc))
            return QualityCheckResult(
                check_name=self.name,
                status=CheckStatus.FAILED,
                severity=self.severity,
                message=f"Check raised exception: {exc}",
                checked_at=datetime.utcnow(),
                records_checked=len(data),
                records_failed=len(data),
            )

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    failed = [row for row in data if row.get(column) is None]
    return len(failed) == 0, failed

return QualityCheck(
    name=f"not_null_{column}",
    description=f"Column '{column}' must not contain null values",
    check_fn=_check,
    severity=CheckSeverity.ERROR,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    seen: dict[Any, int] = {}
    duplicates: list[dict[str, Any]] = []
    for row in data:
        val = row.get(column)
        if val in seen:
            duplicates.append(row)
        seen[val] = seen.get(val, 0) + 1
    return len(duplicates) == 0, duplicates

return QualityCheck(
    name=f"unique_{column}",
    description=f"Column '{column}' must contain unique values",
    check_fn=_check,
    severity=CheckSeverity.ERROR,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    failed = []
    for row in data:
        val = row.get(column)
        if val is None:
            continue
        if min_value is not None and val < min_value:
            failed.append(row)
        elif max_value is not None and val > max_value:
            failed.append(row)
    return len(failed) == 0, failed

bounds = []
if min_value is not None:
    bounds.append(f">= {min_value}")
if max_value is not None:
    bounds.append(f"<= {max_value}")
desc = f"Column '{column}' must be {' and '.join(bounds)}"

return QualityCheck(
    name=f"range_{column}",
    description=desc,
    check_fn=_check,
    severity=CheckSeverity.ERROR,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    if not data:
        return True, []
    now = datetime.utcnow()
    stale = []
    for row in data:
        ts = row.get(timestamp_column)
        if ts is None:
            continue
        if isinstance(ts, str):
            ts = datetime.fromisoformat(ts)
        age = (now - ts).total_seconds() / 3600
        if age > max_age_hours:
            stale.append(row)
    return len(stale) == 0, stale

return QualityCheck(
    name=f"freshness_{timestamp_column}",
    description=f"Column '{timestamp_column}' must not be older than {max_age_hours} hours",
    check_fn=_check,
    severity=CheckSeverity.CRITICAL,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    count = len(data)
    if count < min_rows:
        return False, []
    if max_rows is not None and count > max_rows:
        return False, []
    return True, []

desc = f"Row count must be >= {min_rows}"
if max_rows is not None:
    desc += f" and <= {max_rows}"

return QualityCheck(
    name="row_count",
    description=desc,
    check_fn=_check,
    severity=CheckSeverity.CRITICAL,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    orphans = [row for row in data if row.get(column) not in reference_values]
    return len(orphans) == 0, orphans

return QualityCheck(
    name=f"referential_integrity_{column}",
    description=f"All values in '{column}' must exist in the reference set",
    check_fn=_check,
    severity=CheckSeverity.ERROR,
)

def _check(data: list[dict[str, Any]]) -> tuple[bool, list[dict[str, Any]]]:
    if not data:
        return True, []
    actual_columns = set(data[0].keys())
    missing = set(expected_columns) - actual_columns
    if missing:
        return False, [{"missing_columns": list(missing)}]
    return True, []

return QualityCheck(
    name="schema_check",
    description=f"Data must contain columns: {', '.join(expected_columns)}",
    check_fn=_check,
    severity=CheckSeverity.CRITICAL,
)

undefined

"""Retry and error handling utilities for pipeline resilience."""

import functools
import time
from typing import Any, Callable, Optional, Type

import structlog

logger = structlog.get_logger(__name__)


class PipelineError(Exception):
    """Base exception for all pipeline errors."""

    def __init__(self, message: str, stage: str, details: Optional[dict] = None) -> None:
        super().__init__(message)
        self.stage = stage
        self.details = details or {}


class ExtractionError(PipelineError):
    """Raised when data extraction fails."""

    def __init__(self, message: str, details: Optional[dict] = None) -> None:
        super().__init__(message, stage="extraction", details=details)


class TransformationError(PipelineError):
    """Raised when data transformation fails."""

    def __init__(self, message: str, details: Optional[dict] = None) -> None:
        super().__init__(message, stage="transformation", details=details)


class LoadError(PipelineError):
    """Raised when data loading fails."""

    def __init__(self, message: str, details: Optional[dict] = None) -> None:
        super().__init__(message, stage="load", details=details)


class QualityCheckError(PipelineError):
    """Raised when a critical quality check fails."""

    def __init__(self, message: str, details: Optional[dict] = None) -> None:
        super().__init__(message, stage="quality_check", details=details)


def retry(
    max_attempts: int = 3,
    delay_seconds: float = 1.0,
    backoff_factor: float = 2.0,
    max_delay_seconds: float = 300.0,
    retryable_exceptions: tuple[Type[Exception], ...] = (Exception,),
) -> Callable:
    """Decorator that retries a function with exponential backoff.

    Args:
        max_attempts: Maximum number of attempts before giving up.
        delay_seconds: Initial delay between retries in seconds.
        backoff_factor: Multiplier applied to delay after each retry.
        max_delay_seconds: Maximum delay between retries.
        retryable_exceptions: Tuple of exception types that trigger a retry.

    Returns:
        Decorated function with retry logic.
    """

    def decorator(func: Callable) -> Callable:
        @functools.wraps(func)
        def wrapper(*args: Any, **kwargs: Any) -> Any:
            log = logger.bind(function=func.__name__)
            current_delay = delay_seconds
            last_exception: Optional[Exception] = None

            for attempt in range(1, max_attempts + 1):
                try:
                    return func(*args, **kwargs)
                except retryable_exceptions as exc:
                    last_exception = exc
                    if attempt == max_attempts:
                        log.error(
                            "max_retries_exceeded",
                            attempt=attempt,
                            error=str(exc),
                        )
                        raise
                    log.warning(
                        "retrying",
                        attempt=attempt,
                        max_attempts=max_attempts,
                        delay=current_delay,
                        error=str(exc),
                    )
                    time.sleep(current_delay)
                    current_delay = min(
                        current_delay * backoff_factor,
                        max_delay_seconds,
                    )

            raise last_exception  # Should not reach here, but safety net

        return wrapper

    return decorator

"""Airflow DAG template for orchestrating the data pipeline."""

from datetime import datetime, timedelta
from typing import Any

from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.operators.empty import EmptyOperator
from airflow.utils.task_group import TaskGroup

start = EmptyOperator(task_id="start")
end = EmptyOperator(task_id="end", trigger_rule="none_failed")

# -- Extraction Tasks -------------------------------------------------------
with TaskGroup("extraction") as extraction_group:
    pass  # Generated extraction tasks go here

# -- Quality Checks (Pre-Transform) -----------------------------------------
with TaskGroup("pre_transform_quality") as pre_quality_group:
    pass  # Generated pre-transform quality checks go here

# -- Transformation Tasks ---------------------------------------------------
with TaskGroup("transformation") as transformation_group:
    pass  # Generated transformation tasks go here

# -- Quality Checks (Post-Transform) ----------------------------------------
with TaskGroup("post_transform_quality") as post_quality_group:
    pass  # Generated post-transform quality checks go here

# -- Load Tasks -------------------------------------------------------------
with TaskGroup("loading") as loading_group:
    pass  # Generated load tasks go here

# -- Final Quality Checks ---------------------------------------------------
with TaskGroup("final_quality") as final_quality_group:
    pass  # Generated final quality checks go here

# -- DAG Dependencies -------------------------------------------------------
(
    start
    >> extraction_group
    >> pre_quality_group
    >> transformation_group
    >> post_quality_group
    >> loading_group
    >> final_quality_group
    >> end
)

undefined

undefined

undefined