Lab 005：上下文包越大越好吗？Token 成本与修复成功率实验

实验假设：
  H1：上下文越大，修复成功率越高（可能不对）
  H2：Token 成本与上下文大小线性相关（可能不对，有缓存）
  H3：存在"甜点区间"，超过后成功率不再显著提升（待验证）
  H4：不同类型 Bug 的最佳上下文大小不同（待验证）

# context-pack-configs.yaml
configs:
  small:
    name: "S-最小上下文"
    max_tokens: 5000
    include:
      - error_file           # 报错文件全文
      - stack_trace_files:   # 堆栈指向的文件
          limit: 3
          mode: "snippet"    # 只取报错函数前后 30 行
      - bug_description      # Issue 描述
    exclude:
      - related_modules
      - git_history
      - test_files
      - project_rules

  medium:
    name: "M-标准上下文"
    max_tokens: 20000
    include:
      - error_file
      - stack_trace_files:
          limit: 3
          mode: "full"       # 完整文件
      - bug_description
      - related_modules:     # 同模块文件
          limit: 5
          strategy: "import_graph"
      - test_files:          # 相关测试
          limit: 3
      - project_rules:       # CLAUDE.md
          files: ["CLAUDE.md", ".cursorrules"]
    exclude:
      - git_history

  large:
    name: "L-全量上下文"
    max_tokens: 80000
    include:
      - error_file
      - stack_trace_files:
          limit: 10
          mode: "full"
      - bug_description
      - related_modules:
          limit: 20
          strategy: "import_graph"
      - test_files:
          limit: 10
      - project_rules:
          files: ["CLAUDE.md", ".cursorrules", "docs/**/*.md"]
      - git_history:         # 最近 20 条 commit
          limit: 20
          scope: "affected_files"
      - linked_issues:       # 关联 Issue
          limit: 5

# context_size_experiment.py
import json
import time
import subprocess
from dataclasses import dataclass, asdict

@dataclass
class ExperimentResult:
    bug_id: str
    bug_type: str
    config: str           # "small" / "medium" / "large"
    tokens_used: int
    cost_usd: float
    time_seconds: float
    tests_passed: int
    tests_total: int
    fix_correct: bool
    diff_similarity: float  # 与标准修复的相似度 0-1
    code_quality_score: float  # 0-1
    composite_score: float

def run_single(bug_id: str, config_name: str, agent_runner) -> ExperimentResult:
    """对单个 Bug 用指定上下文配置跑一次"""
    # 1. 加载上下文包
    context = load_context_pack(bug_id, config_name)
    
    # 2. 记录开始时间和 Token
    start = time.time()
    token_counter = TokenCounter()
    
    # 3. 调用 Agent 修复
    patch = agent_runner.fix(
        context=context,
        bug_description=load_bug_desc(bug_id),
        token_callback=token_counter.on_token,
    )
    
    elapsed = time.time() - start
    tokens = token_counter.total
    
    # 4. 应用补丁并运行测试
    apply_patch(patch)
    test_result = run_tests(get_test_suite(bug_id))
    
    # 5. 计算评分
    similarity = compute_diff_similarity(patch, load_golden_patch(bug_id))
    quality = run_lint_check(patch)
    correct = test_result.passed == test_result.total
    
    # 6. 计算综合得分
    cost_score = max(0, 1 - tokens / 80000)  # Token 越少分越高
    composite = (correct * 0.4 + similarity * 0.25 + 
                 quality * 0.2 + cost_score * 0.15)
    
    # 7. 回滚补丁（为下一轮准备）
    revert_patch()
    
    return ExperimentResult(
        bug_id=bug_id,
        bug_type=classify_bug(bug_id),
        config=config_name,
        tokens_used=tokens,
        cost_usd=tokens * PRICE_PER_TOKEN,
        time_seconds=elapsed,
        tests_passed=test_result.passed,
        tests_total=test_result.total,
        fix_correct=correct,
        diff_similarity=similarity,
        code_quality_score=quality,
        composite_score=composite,
    )

def run_experiment(bug_ids: list[str], agent_runner):
    """运行完整实验：每个 Bug × 三种配置"""
    configs = ["small", "medium", "large"]
    results = []
    
    for bug_id in bug_ids:
        for config in configs:
            result = run_single(bug_id, config, agent_runner)
            results.append(asdict(result))
            print(f"[{bug_id}] {config}: score={result.composite_score:.2f} "
                  f"tokens={result.tokens_used} correct={result.fix_correct}")
    
    # 保存结果
    with open("experiment-results.json", "w") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)
    
    return results

成功率曲线（按上下文大小）：

100% ┤                                          ●━━━━ L
     │                                    ╱─────
 80% ┤                              ●────●      M
     │                         ╱────╯
 60% ┤                    ╱────╯
     │               ╱────╯
 40% ┤          ╱────╯                ●━━━━━━━ S
     │     ╱────╯
 20% ┤────╯
     │
  0% ┼──────┬──────┬──────┬──────┬───
     S(3K)  M(12K) L(48K)
              上下文 Token 数

# dynamic-context-routing.yaml
strategy: "start_medium_then_scale"

rules:
  - condition: "bug_type in ['配置问题', '类型错误']"
    action: "use_small"
    reason: "这类 Bug 通常只看报错文件就能修"

  - condition: "bug_type in ['并发问题', '集成问题']"
    action: "use_large"
    reason: "需要跨模块上下文才能理解竞态和协议问题"

  - condition: "medium_failed_and_retry_budget > 0"
    action: "upgrade_to_large"
    reason: "M 修不好，升级 L 再试一次"

  - condition: "default"
    action: "use_medium"
    reason: "性价比最高的默认选择"

# cost_guard.py
CONTEXT_BUDGET = {
    "daily": 5.0,       # 每日 Token 预算 $5
    "per_task": 0.50,   # 单任务预算 $0.50
    "alert_threshold": 0.80,  # 用到 80% 预算时告警
}

def check_budget(tokens_used: int, config: str) -> bool:
    """检查是否超出预算，超出不执行"""
    cost = tokens_used * PRICE_PER_TOKEN
    if cost > CONTEXT_BUDGET["per_task"]:
        log_alert(f"Task would exceed per-task budget: ${cost:.2f}")
        return False
    return True