class ResearchPipeline:
    """
    D.E. Shaw's approach: systematic research with reproducibility.
    Every experiment is tracked, versioned, and reproducible.
    """
    
    def __init__(self, experiment_tracker, data_warehouse, compute_cluster):
        self.tracker = experiment_tracker
        self.data = data_warehouse
        self.compute = compute_cluster
    
    def run_experiment(self,
                       hypothesis: Hypothesis,
                       config: ExperimentConfig) -> ExperimentResult:
        """
        Run a single experiment with full tracking.
        """
        # Create experiment record
        experiment_id = self.tracker.create_experiment(
            hypothesis=hypothesis.description,
            config=config.to_dict(),
            git_commit=get_git_commit(),
            data_version=self.data.get_version()
        )
        
        try:
            # Load data with point-in-time correctness
            data = self.data.load(
                universe=config.universe,
                start_date=config.start_date,
                end_date=config.end_date,
                as_of_date=config.as_of_date  # Prevent lookahead
            )
            
            # Validate data quality
            quality_report = self.validate_data(data)
            self.tracker.log_artifact(experiment_id, 'data_quality', quality_report)
            
            # Run the actual analysis
            result = hypothesis.evaluate(data, config)
            
            # Compute statistical significance
            significance = self.assess_significance(result, config)
            
            # Log results
            self.tracker.log_metrics(experiment_id, {
                'sharpe_ratio': result.sharpe_ratio,
                'information_ratio': result.information_ratio,
                't_statistic': significance.t_stat,
                'p_value': significance.p_value,
                'num_observations': result.n_obs
            })
            
            return ExperimentResult(
                experiment_id=experiment_id,
                hypothesis=hypothesis,
                result=result,
                significance=significance,
                reproducible=True
            )
            
        except Exception as e:
            self.tracker.log_failure(experiment_id, str(e))
            raise
    
    def run_hypothesis_suite(self, 
                             hypotheses: List[Hypothesis],
                             config: ExperimentConfig) -> SuiteResult:
        """
        Run multiple hypotheses and correct for multiple testing.
        """
        results = []
        
        for hypothesis in hypotheses:
            result = self.run_experiment(hypothesis, config)
            results.append(result)
        
        # Apply Benjamini-Hochberg FDR correction
        corrected = self.apply_fdr_correction(results)
        
        return SuiteResult(
            results=corrected,
            significant_count=sum(1 for r in corrected if r.is_significant),
            total_count=len(corrected)
        )

class RiskModel:
    """
    D.E. Shaw's risk approach: understand and control risk at multiple levels.
    """
    
    def __init__(self, factor_returns, factor_covariance, specific_risk):
        self.factor_returns = factor_returns  # Historical factor returns
        self.factor_cov = factor_covariance   # Factor covariance matrix
        self.specific_risk = specific_risk    # Idiosyncratic risk by asset
    
    def estimate_portfolio_risk(self,
                                 positions: pd.Series,
                                 factor_exposures: pd.DataFrame) -> RiskEstimate:
        """
        Decompose portfolio risk into systematic and idiosyncratic components.
        """
        # Factor risk: w' * B * Σ_f * B' * w
        portfolio_exposures = factor_exposures.T @ positions
        factor_var = portfolio_exposures @ self.factor_cov @ portfolio_exposures
        
        # Specific risk: Σ(w_i^2 * σ_i^2)
        specific_var = (positions ** 2 * self.specific_risk ** 2).sum()
        
        # Total risk
        total_var = factor_var + specific_var
        
        return RiskEstimate(
            total_volatility=np.sqrt(total_var * 252),  # Annualized
            factor_volatility=np.sqrt(factor_var * 252),
            specific_volatility=np.sqrt(specific_var * 252),
            factor_contribution=self.calculate_factor_contributions(
                positions, factor_exposures
            )
        )
    
    def calculate_factor_contributions(self, positions, factor_exposures):
        """
        Break down risk by factor for attribution.
        """
        portfolio_exposures = factor_exposures.T @ positions
        
        contributions = {}
        for factor in self.factor_cov.columns:
            # Marginal contribution to risk
            factor_exposure = portfolio_exposures[factor]
            factor_vol = np.sqrt(self.factor_cov.loc[factor, factor])
            contributions[factor] = {
                'exposure': factor_exposure,
                'volatility': factor_vol,
                'contribution': factor_exposure * factor_vol
            }
        
        return contributions
    
    def stress_test(self, 
                    positions: pd.Series,
                    scenarios: Dict[str, Dict[str, float]]) -> Dict[str, float]:
        """
        Apply historical or hypothetical stress scenarios.
        """
        results = {}
        
        for scenario_name, factor_shocks in scenarios.items():
            pnl = 0.0
            
            for factor, shock in factor_shocks.items():
                factor_exposure = self.get_portfolio_exposure(positions, factor)
                pnl += factor_exposure * shock
            
            results[scenario_name] = pnl
        
        return results

class StrategyFramework:
    """
    D.E. Shaw's modular strategy architecture.
    Strategies are composed from reusable components.
    """
    
    def __init__(self):
        self.alpha_models = {}
        self.risk_models = {}
        self.execution_models = {}
        self.portfolio_constructors = {}
    
    def register_alpha_model(self, name: str, model: AlphaModel):
        """Alpha models generate return predictions."""
        self.alpha_models[name] = model
    
    def register_risk_model(self, name: str, model: RiskModel):
        """Risk models estimate covariances and factor exposures."""
        self.risk_models[name] = model
    
    def create_strategy(self, config: StrategyConfig) -> Strategy:
        """
        Compose a strategy from registered components.
        """
        alpha = self.alpha_models[config.alpha_model]
        risk = self.risk_models[config.risk_model]
        execution = self.execution_models[config.execution_model]
        constructor = self.portfolio_constructors[config.portfolio_constructor]
        
        return ComposedStrategy(
            alpha_model=alpha,
            risk_model=risk,
            execution_model=execution,
            portfolio_constructor=constructor,
            constraints=config.constraints,
            risk_limits=config.risk_limits
        )


class ComposedStrategy:
    """
    A strategy composed from modular components.
    """
    
    def __init__(self, alpha_model, risk_model, execution_model,
                 portfolio_constructor, constraints, risk_limits):
        self.alpha = alpha_model
        self.risk = risk_model
        self.execution = execution_model
        self.constructor = portfolio_constructor
        self.constraints = constraints
        self.risk_limits = risk_limits
    
    def generate_trades(self, 
                        current_positions: pd.Series,
                        market_data: MarketData) -> List[Trade]:
        """
        Full strategy pipeline: alpha → portfolio → trades.
        """
        # 1. Generate alpha signals
        alpha_scores = self.alpha.predict(market_data)
        
        # 2. Estimate risk
        risk_estimate = self.risk.estimate(market_data)
        
        # 3. Construct optimal portfolio
        target_positions = self.constructor.optimize(
            alpha_scores=alpha_scores,
            risk_model=risk_estimate,
            current_positions=current_positions,
            constraints=self.constraints,
            risk_limits=self.risk_limits
        )
        
        # 4. Generate trades to move from current to target
        trades = self.calculate_trades(current_positions, target_positions)
        
        # 5. Optimize execution
        scheduled_trades = self.execution.schedule(trades, market_data)
        
        return scheduled_trades

class PortfolioOptimizer:
    """
    Mean-variance optimization with realistic constraints.
    """
    
    def optimize(self,
                 alpha: pd.Series,
                 covariance: pd.DataFrame,
                 current_positions: pd.Series,
                 constraints: ConstraintSet) -> pd.Series:
        """
        Solve the quadratic programming problem:
        
        max: α'w - λ/2 * w'Σw - γ * ||w - w_0||^2
        s.t.: constraints
        """
        n = len(alpha)
        
        # Objective: maximize alpha, minimize risk, minimize turnover
        P = constraints.risk_aversion * covariance.values
        P += constraints.turnover_aversion * np.eye(n)
        q = -alpha.values + constraints.turnover_aversion * current_positions.values
        
        # Constraints
        G, h = self.build_inequality_constraints(constraints, n)
        A, b = self.build_equality_constraints(constraints, n)
        
        # Solve
        solution = qp_solve(P, q, G, h, A, b)
        
        return pd.Series(solution, index=alpha.index)
    
    def build_inequality_constraints(self, constraints, n):
        """
        Build inequality constraints: Gx <= h
        - Long-only: -w <= 0
        - Position limits: w <= max_position
        - Sector limits: Σw_sector <= max_sector
        """
        G_list = []
        h_list = []
        
        if constraints.long_only:
            G_list.append(-np.eye(n))
            h_list.append(np.zeros(n))
        
        if constraints.max_position:
            G_list.append(np.eye(n))
            h_list.append(np.full(n, constraints.max_position))
        
        for sector, (assets, max_weight) in constraints.sector_limits.items():
            row = np.zeros(n)
            row[assets] = 1.0
            G_list.append(row.reshape(1, -1))
            h_list.append(np.array([max_weight]))
        
        return np.vstack(G_list), np.concatenate(h_list)
    
    def build_equality_constraints(self, constraints, n):
        """
        Build equality constraints: Ax = b
        - Fully invested: Σw = 1
        - Dollar neutral: Σw = 0
        """
        A_list = []
        b_list = []
        
        if constraints.fully_invested:
            A_list.append(np.ones((1, n)))
            b_list.append(np.array([1.0]))
        
        if constraints.dollar_neutral:
            A_list.append(np.ones((1, n)))
            b_list.append(np.array([0.0]))
        
        if A_list:
            return np.vstack(A_list), np.concatenate(b_list)
        return None, None