┌─────────────────┐
│  Claude Code    │ (AI Assistant)
│  (or any LLM)   │
└────────┬────────┘
         │ MCP Protocol
         │ (JSON-RPC via stdio)
         ▼
┌─────────────────┐
│   MCP Server    │ (This project)
│   (Node.js)     │
└────────┬────────┘
         │
         ├─────▶ PostgreSQL DB (TestOps data)
         ├─────▶ AI Providers (Anthropic, OpenAI, etc.)
         ├─────▶ Vector DB (Weaviate - semantic search)
         └─────▶ Cache (Redis - optional)

mcp-server/
├── src/
│   ├── index.ts          # MCP server entry point
│   ├── types.ts          # TypeScript type definitions
│   ├── db.ts             # Database connection & queries
│   └── tools/            # Tool implementations
│       ├── analyze.ts    # Single failure analysis
│       ├── batch.ts      # Batch failure analysis
│       ├── knowledge.ts  # Knowledge base tools
│       └── stats.ts      # Statistics tools
├── dist/                 # Compiled JavaScript (after build)
├── package.json          # Dependencies & scripts
├── tsconfig.json         # TypeScript configuration
├── .env                  # Environment variables
├── README.md             # User documentation
└── SKILL.md              # This file

1. Claude Code sends request:
   {
     "method": "tools/call",
     "params": {
       "name": "testops_analyze_failure",
       "arguments": { "testName": "...", "errorMessage": "..." }
     }
   }

2. MCP Server receives request → routes to tool handler

3. Tool executes:
   - Validates input (Zod schema)
   - Queries database
   - Calls AI services (if needed)
   - Returns structured result

4. MCP Server sends response:
   {
     "content": [{
       "type": "text",
       "text": "{ ... JSON result ... }"
     }]
   }

5. Claude Code interprets result → presents to user

{
  testName: string;        // Required
  errorMessage: string;    // Required
  stackTrace?: string;     // Optional
  logs?: string;           // Optional (last 5000 chars recommended)
  pipeline?: string;       // Optional
  branch?: string;         // Optional
}

{
  categorization: {
    category: string;      // INFRASTRUCTURE_ISSUE, APPLICATION_BUG, etc.
    subcategory?: string;
    confidence: number;    // 0-1
    reasoning: string;
  };
  similarFailures: Array<{
    id: string;
    testName: string;
    similarity: number;    // 0-1
    resolution?: string;   // How it was fixed
    resolvedBy?: string;
    ticketUrl?: string;
  }>;
  logSummary?: {
    summary: string;
    rootCause: string;
    technicalDetails: string;
    suggestedFix: string;
  };
  estimatedCostUSD: number;
}

// Quick categorization (no logs)
{
  testName: "payment-test",
  errorMessage: "Timeout waiting for response",
}
// Cost: ~$0.01

// Full analysis with logs
{
  testName: "payment-test",
  errorMessage: "Timeout waiting for response",
  logs: "... last 5000 chars ...",
}
// Cost: ~$0.05

{
  testRunId?: string;      // Option 1: Load from test run
  failures?: Array<{       // Option 2: Provide failures directly
    testName: string;
    errorMessage: string;
    stackTrace?: string;
    logs?: string;
  }>;
  maxFailures?: number;    // Default: 20
  priorityThreshold?: number; // Default: 0.7
}

{
  totalFailures: number;
  analyzedFailures: number;
  summary: string;         // Human-readable summary
  patterns: Array<{
    category: string;
    count: number;
    examples: string[];    // Example test names
  }>;
  priorities: {
    high: TestFailure[];   // Fix these first
    medium: TestFailure[];
    low: TestFailure[];    // Flaky tests, low confidence
  };
  estimatedTotalCostUSD: number;
}

// Analyze a test run
{
  testRunId: "123e4567-e89b-12d3-a456-426614174000"
}

// Analyze specific failures
{
  failures: [
    { testName: "test1", errorMessage: "..." },
    { testName: "test2", errorMessage: "..." },
  ],
  maxFailures: 15
}

{
  query: string;           // Search text
  category?: string;       // Optional filter
  limit?: number;          // Default: 10
  onlyResolved?: boolean;  // Default: true
}

Array<{
  id: string;
  testName: string;
  errorMessage: string;
  category: string;
  resolution: string;      // How it was fixed!
  resolvedBy: string;
  ticketUrl?: string;
  createdAt: Date;
  similarity?: number;     // If using vector search
}>

// Search by error message
{
  query: "database connection refused"
}

// Search with category filter
{
  query: "timeout",
  category: "INFRASTRUCTURE_ISSUE",
  limit: 5
}

{
  testName: string;        // Required
  errorMessage: string;    // Required
  stackTrace?: string;
  category: string;        // Required - see categories below
  resolution: string;      // Required - be specific!
  resolvedBy: string;      // Required - email or username
  ticketUrl?: string;      // Optional - link to issue/PR
  pipeline?: string;
  branch?: string;
}

{
  id: string;
  message: string;
}

{
  testName: "checkout-payment-test",
  errorMessage: "Connection timeout to payment gateway",
  category: "INFRASTRUCTURE_ISSUE",
  resolution: "Increased timeout from 5s to 15s in config/test.yml. Payment gateway has 10s latency in staging.",
  resolvedBy: "alice@example.com",
  ticketUrl: "https://github.com/org/repo/pull/456",
  pipeline: "e2e-tests"
}

{
  pipelineId?: string;     // Optional - omit for all pipelines
  limit?: number;          // Default: 10 recent runs
  daysBack?: number;       // Default: 30 days
}

Array<{
  id: string;
  name: string;
  type: string;            // GITHUB_ACTIONS, JENKINS, etc.
  recentRuns: Array<{
    id: string;
    status: string;
    startedAt: Date;
    duration?: number;
    failed: number;
  }>;
  successRate: number;     // Percentage
  avgDuration: number;     // Milliseconds
  commonFailures: Array<{
    testName: string;
    count: number;
    lastOccurrence: Date;
  }>;
}>

{
  testName: string;        // Required
  daysBack?: number;       // Default: 90
  limit?: number;          // Default: 20
}

{
  testName: string;
  totalRuns: number;
  failures: number;
  flakinessScore: number;  // 0 = stable, 1 = very flaky
  failurePatterns: Array<{
    errorMessage: string;
    count: number;
    lastOccurrence: Date;
  }>;
  recentRuns: Array<{
    id: string;
    status: string;
    timestamp: Date;
    pipeline: string;
    branch: string;
  }>;
}

{
  startDate?: string;      // ISO format, default: 30 days ago
  endDate?: string;        // ISO format, default: now
}

{
  period: {
    start: Date;
    end: Date;
  };
  totalCostUSD: number;
  breakdown: Array<{
    feature: string;       // categorization, rca_matching, etc.
    calls: number;
    totalTokens: number;
    costUSD: number;
  }>;
  topExpensiveOperations: Array<{
    timestamp: Date;
    feature: string;
    costUSD: number;
    tokens: number;
  }>;
}

{
  healthy: boolean;
  services: {
    database?: {
      healthy: boolean;
      error?: string;
    };
    aiProvider?: {
      healthy: boolean;
      name?: string;
      error?: string;
    };
    vectorDB?: {
      healthy: boolean;
      error?: string;
    };
    cache?: {
      healthy: boolean;
      stats?: any;
    };
  };
}

❌ BAD: Analyze every failure
User: "Test X failed"
→ testops_analyze_failure ($0.05)

✅ GOOD: Search first, analyze if needed
User: "Test X failed"
→ testops_search_knowledge ($0.001)
   ↓
   Found solution? → Use it (saved $0.049)
   Not found? → testops_analyze_failure ($0.05)

❌ BAD: Analyze 10 failures individually
10 × testops_analyze_failure = $0.50

✅ GOOD: Batch analyze
testops_batch_analyze = $0.10

**Savings:** $0.40 (80%)

❌ BAD: Include entire 500KB log file
→ ~125,000 tokens
→ ~$0.40 per analysis

✅ GOOD: Include last 5000 chars
→ ~1,250 tokens
→ ~$0.004 per analysis

**Savings:** $0.396 per analysis (99%)

Scenario: "Login timeout" failure occurs monthly

Without knowledge base:
- 12 months × $0.05 = $0.60/year

With knowledge base:
- First occurrence: $0.05 (analysis) + $0.001 (add to KB)
- Next 11 occurrences: 11 × $0.001 (search) = $0.011
- Total: $0.061/year

**Savings:** $0.539/year (90%) for just one recurring issue

export interface MyToolInput {
  param1: string;
  param2?: number;
}

export interface MyToolOutput {
  result: string;
  metadata: any;
}

import { z } from 'zod';

const MyToolInputSchema = z.object({
  param1: z.string(),
  param2: z.number().optional(),
});

export async function myTool(input: MyToolInput): Promise<MyToolOutput> {
  const validated = MyToolInputSchema.parse(input);

  // Implementation

  return {
    result: '...',
    metadata: {},
  };
}

export const myToolDefinition = {
  name: 'testops_my_tool',
  description: '...',
  inputSchema: {
    type: 'object',
    properties: {
      param1: {
        type: 'string',
        description: '...',
      },
    },
    required: ['param1'],
  },
};

import { myTool, myToolDefinition } from './tools/mytool.js';

const tools: Tool[] = [
  // ... existing tools
  myToolDefinition as Tool,
];

// In CallToolRequestSchema handler
case 'testops_my_tool':
  result = await myTool(args);
  break;

npm run build
npm run inspector
# Use MCP inspector to test the tool

// ✅ Good - prevents SQL injection
await query('SELECT * FROM tests WHERE name = $1', [testName]);

// ❌ Bad - vulnerable to SQL injection
await query(`SELECT * FROM tests WHERE name = '${testName}'`);

import { transaction } from '../db.js';

await transaction(async (client) => {
  await client.query('INSERT INTO failures ...');
  await client.query('UPDATE stats ...');
  // Both succeed or both roll back
});

try {
  const results = await query('...');
  return results;
} catch (error) {
  console.error('Query failed:', error);
  throw new Error(`Failed to fetch data: ${error}`);
}

npm run dev

npm run inspector
# Opens interactive web UI for testing tools

User: "Why did test X fail?"

CORRECT FLOW:
1. testops_search_knowledge({ query: "test X" })
   ↓
   Found? → Present solution
   Not found? → Continue to step 2

2. testops_analyze_failure({ testName: "test X", ... })
   ↓
   Present analysis + ask if they want to add to KB

3. (After resolution) testops_add_knowledge(...)

User: "CI run failed with 15 test failures"

CORRECT FLOW:
1. testops_batch_analyze({ testRunId: "..." })
   ↓
   Get summary + priorities

2. Focus on high-priority failures
   ↓
   testops_search_knowledge for each

3. Deep dive if needed
   ↓
   testops_analyze_failure (with logs) for complex cases

❌ BAD:
testops_analyze_failure({
  testName: "...",
  errorMessage: "...",
  logs: entireLogFile  // 500KB
})

✅ GOOD:
testops_analyze_failure({
  testName: "...",
  errorMessage: "...",
  logs: lastNLines(logFile, 200)  // ~5000 chars
})

After helping user resolve a failure:

"I've helped you fix this issue. Would you like me to add it to
the knowledge base so you (or teammates) can find the solution
instantly next time?"

→ testops_add_knowledge(...)

const InputSchema = z.object({
  required: z.string(),
  optional: z.number().optional(),
});

export async function myTool(input: any) {
  const validated = InputSchema.parse(input);  // Throws if invalid
  // ... use validated data
}

try {
  // ... operation
} catch (error) {
  console.error('Operation failed:', error);
  throw new Error(
    `Failed to perform operation: ${error instanceof Error ? error.message : 'Unknown error'}`
  );
}

let cost = 0;

// Track embedding generation
cost += 0.001;

// Track AI completion
const tokens = estimateTokens(prompt);
cost += (tokens / 1000000) * 3;  // $3 per million tokens

return {
  ...result,
  estimatedCostUSD: Math.round(cost * 1000) / 1000,
};

-- In backend/prisma/schema.prisma
@@index([testName])
@@index([status, createdAt])

await query(`
  SELECT * FROM tests
  WHERE status = 'FAILED'
  ORDER BY created_at DESC
  LIMIT 100  -- Always limit!
`, []);

⚠️  WARNING: DATABASE_URL not set
Error: Connection string is not set

cp .env.example .env
# Edit .env and set DATABASE_URL
DATABASE_URL=postgresql://user:pass@localhost:5432/testops

Error: relation "failure_records" does not exist

cd ../backend
npm run db:migrate

⚠️  WARNING: AI features are disabled

# In .env
AI_ENABLED=true
AI_PROVIDER=anthropic
ANTHROPIC_API_KEY=sk-ant-...

Error: connect ETIMEDOUT

Error: Cannot find module './tools/analyze.js'

npm run build

LOG_LEVEL=debug npm run dev

# In one terminal
npm run dev

# In another terminal
echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | node dist/index.js

// src/db.ts
export async function testConnection() {
  const client = getPool();
  const result = await client.query('SELECT version()');
  console.log('Database version:', result.rows[0]);
}

function estimateTokens(text: string): number {
  return Math.ceil(text.length / 4);
}

console.log('Estimated tokens:', estimateTokens(inputText));

model FailureRecord {
  // ...
  @@index([testName])
  @@index([category])
  @@index([resolvedAt])
}

// ❌ Bad - returns all rows
SELECT * FROM test_results

// ✅ Good - limits results
SELECT * FROM test_results
ORDER BY created_at DESC
LIMIT 100

// ❌ Bad - fetch all then count in JS
const rows = await query('SELECT * FROM tests');
const count = rows.length;

// ✅ Good - count in database
const result = await query('SELECT COUNT(*) as count FROM tests');
const count = result[0].count;

const cache = new Map<string, { data: any; expires: number }>();

function getCached<T>(key: string): T | null {
  const entry = cache.get(key);
  if (entry && entry.expires > Date.now()) {
    return entry.data as T;
  }
  return null;
}

function setCache(key: string, data: any, ttlMs: number): void {
  cache.set(key, {
    data,
    expires: Date.now() + ttlMs,
  });
}

// ❌ Bad - N queries
for (const test of tests) {
  await query('SELECT * FROM results WHERE test_name = $1', [test.name]);
}

// ✅ Good - 1 query with IN clause
const testNames = tests.map(t => t.name);
await query(
  'SELECT * FROM results WHERE test_name = ANY($1)',
  [testNames]
);

// ✅ Good - use Zod for validation
const InputSchema = z.object({
  testName: z.string().max(500),  // Limit length
  errorMessage: z.string().max(5000),
});

const validated = InputSchema.parse(input);

// ✅ Good
await query('SELECT * FROM tests WHERE id = $1', [testId]);

// ❌ Bad
await query(`SELECT * FROM tests WHERE id = '${testId}'`);

# ✅ Good - in .env (gitignored)
ANTHROPIC_API_KEY=sk-ant-...

# ❌ Bad - hardcoded
const apiKey = 'sk-ant-...';

// ✅ Good
throw new Error('Failed to fetch data');

// ❌ Bad
throw new Error(`Failed: ${DATABASE_URL} connection failed`);