┌─────────────────────────────────────────────────────────────────┐
│                   MODEL SELECTION FRAMEWORK                      │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  1. TASK ANALYSIS                                               │
│     What are the core capabilities needed?                      │
│     ├── Text Generation → LLM                                   │
│     ├── Classification → Traditional ML / Small LM              │
│     ├── Code Generation → Code-specialized LLM                  │
│     ├── Vision → Multimodal / Vision Model                      │
│     ├── Embeddings → Embedding Model                            │
│     └── Structured Output → Instruction-tuned LLM               │
│                                                                  │
│  2. REQUIREMENTS MAPPING                                         │
│     ├── Quality: Accuracy, coherence, factuality                │
│     ├── Latency: Real-time vs batch                             │
│     ├── Cost: Per-token, per-request budgets                    │
│     ├── Privacy: Data residency, local deployment               │
│     └── Scale: Requests per second, concurrent users            │
│                                                                  │
│  3. MODEL EVALUATION                                             │
│     ├── Benchmark analysis                                       │
│     ├── Task-specific testing                                   │
│     └── Cost-performance optimization                           │
│                                                                  │
│  4. DEPLOYMENT PLANNING                                          │
│     ├── Cloud API vs self-hosted                                │
│     ├── Hardware requirements                                   │
│     └── Scaling strategy                                        │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

public class ModelEvaluator
{
    public async Task<EvaluationReport> EvaluateModels(
        List<ModelConfig> candidates,
        EvaluationDataset dataset,
        CancellationToken ct)
    {
        var results = new Dictionary<string, ModelMetrics>();

        foreach (var model in candidates)
        {
            var metrics = new ModelMetrics
            {
                ModelId = model.Id,
                Provider = model.Provider
            };

            // Run task-specific tests
            foreach (var testCase in dataset.TestCases)
            {
                var startTime = Stopwatch.StartNew();

                var response = await CallModel(model, testCase.Prompt, ct);

                startTime.Stop();

                metrics.AddResult(new TestResult
                {
                    TestId = testCase.Id,
                    LatencyMs = startTime.ElapsedMilliseconds,
                    InputTokens = CountTokens(testCase.Prompt),
                    OutputTokens = CountTokens(response),
                    Score = await EvaluateResponse(response, testCase.Expected),
                    Cost = CalculateCost(model, testCase.Prompt, response)
                });
            }

            results[model.Id] = metrics;
        }

        return new EvaluationReport
        {
            Results = results,
            Recommendation = SelectBestModel(results, dataset.Requirements)
        };
    }

    private ModelRecommendation SelectBestModel(
        Dictionary<string, ModelMetrics> results,
        Requirements requirements)
    {
        // Score based on requirements weights
        var scores = results.Select(r => new
        {
            Model = r.Key,
            Score = CalculateWeightedScore(r.Value, requirements)
        }).OrderByDescending(s => s.Score);

        return new ModelRecommendation
        {
            Primary = scores.First().Model,
            Fallback = scores.Skip(1).FirstOrDefault()?.Model,
            Reasoning = GenerateReasoning(scores, requirements)
        };
    }
}

public class ModelCostCalculator
{
    public CostProjection CalculateMonthlyCost(
        ModelConfig model,
        UsageEstimate usage)
    {
        var inputCost = usage.MonthlyInputTokens / 1_000_000m
            * model.InputPricePerMillion;

        var outputCost = usage.MonthlyOutputTokens / 1_000_000m
            * model.OutputPricePerMillion;

        var cachedSavings = usage.CacheHitRate * inputCost
            * model.CacheDiscount;

        return new CostProjection
        {
            GrossInputCost = inputCost,
            GrossOutputCost = outputCost,
            CacheSavings = cachedSavings,
            NetMonthlyCost = inputCost + outputCost - cachedSavings,
            CostPerRequest = (inputCost + outputCost - cachedSavings)
                / usage.MonthlyRequests
        };
    }

    public ModelComparison CompareModels(
        List<ModelConfig> models,
        UsageEstimate usage,
        QualityRequirements requirements)
    {
        var comparisons = models.Select(m => new
        {
            Model = m,
            Cost = CalculateMonthlyCost(m, usage),
            MeetsRequirements = EvaluateQuality(m, requirements)
        }).Where(c => c.MeetsRequirements)
          .OrderBy(c => c.Cost.NetMonthlyCost)
          .ToList();

        return new ModelComparison
        {
            CheapestQualified = comparisons.FirstOrDefault()?.Model,
            AllOptions = comparisons,
            Savings = CalculateSavings(comparisons)
        };
    }
}

## Fine-Tuning Decision: [Use Case]

### Current State (Prompting)
- Prompt tokens: [X] tokens
- Output quality: [Score]
- Cost per request: $[X]
- Monthly cost: $[X]

### Projected State (Fine-Tuned)
- Prompt tokens: [Y] tokens (reduced)
- Output quality: [Score] (maintained/improved)
- Fine-tuning cost: $[X] (one-time)
- Inference cost per request: $[Y]
- Monthly cost: $[Y]

### Break-Even Analysis
- Monthly savings: $[X]
- Break-even: [N] months
- 12-month ROI: [X]%

### Recommendation
[Fine-tune / Continue prompting / Hybrid approach]

# Model Selection: [Project Name]

## Task Requirements
- **Primary Use Case**: [Description]
- **Quality Requirements**: [Accuracy/coherence targets]
- **Latency Requirements**: [P95 target]
- **Volume**: [Requests/day]
- **Budget**: [Monthly budget]

## Evaluation Results

| Model | Quality Score | P95 Latency | Monthly Cost | Notes |
|-------|--------------|-------------|--------------|-------|
| [Model A] | [Score] | [ms] | $[X] | [Notes] |
| [Model B] | [Score] | [ms] | $[X] | [Notes] |
| [Model C] | [Score] | [ms] | $[X] | [Notes] |

## Recommendation

**Primary Model**: [Model]
- Rationale: [Why this model]

**Fallback Model**: [Model]
- Use when: [Conditions]

**Cost Projection**: $[X]/month

## Implementation Notes
- [Deployment approach]
- [Monitoring strategy]
- [Scaling considerations]