AgentSkillsCN

chatbot-analytics

实施 AI 聊天机器人分析和对话监控。在添加对话指标、跟踪 AI 使用情况、衡量用户与聊天互动,或构建对话仪表板时使用。启用 AI 分析、令牌跟踪、对话分类和聊天性能。

SKILL.md
--- frontmatter
name: chatbot-analytics
description: Implement AI chatbot analytics and conversation monitoring. Use when adding conversation metrics, tracking AI usage, measuring user engagement with chat, or building conversation dashboards. Activates for AI analytics, token tracking, conversation categorization, and chat performance.
allowed-tools: Read,Write,Edit,Bash(npm:*,npx:*)
category: Data & Analytics
tags:
  - analytics
  - chatbot
  - ai-metrics

AI Chatbot Analytics

This skill helps you implement analytics for the AI coaching chat feature while maintaining HIPAA compliance.

Core Metrics to Track

Based on industry best practices, track these 13 key metrics:

MetricDescriptionHIPAA Safe?
Total SessionsNumber of chat sessionsYes
Avg Messages/SessionMessages per conversationYes
Avg Session DurationTime spent in chatYes
Engagement Rate% users who use chatYes
Completion RateSessions ended naturallyYes
Abandonment RateSessions ended earlyYes
Response TimeAI response latencyYes
Token UsageTotal/avg tokens consumedYes
Error RateFailed responsesYes
Fallback Rate"I don't understand" responsesYes
Topic CategoriesWhat users discussMetadata only
Sentiment TrendEmotional directionDerived only
Crisis TriggersEmergency detectionMetadata only

HIPAA-Compliant Analytics

What to Track

typescript
// Conversation metadata (SAFE)
interface ConversationAnalytics {
  id: string;
  conversationId: string;
  userId: string;  // For aggregation, not individual tracking
  startedAt: Date;
  endedAt: Date | null;
  messageCount: number;
  userMessageCount: number;
  aiMessageCount: number;
  totalTokens: number;
  inputTokens: number;
  outputTokens: number;
  category: string;  // Derived from metadata flags
  outcome: 'completed' | 'abandoned' | 'error' | 'crisis_escalated';
  avgResponseTime: number;
  hadFallback: boolean;
}

What NOT to Track

typescript
// NEVER store these in analytics
interface PROHIBITED {
  messageContent: string;      // PHI
  userQuery: string;           // PHI
  aiResponse: string;          // PHI
  specificTopics: string[];    // Could reveal health info
  exactSentiment: 'sad';       // Could reveal mental state
}

Implementation Pattern

Tracking Conversation Start

typescript
// src/lib/ai/analytics.ts
export async function trackConversationStart(
  conversationId: string,
  userId: string
): Promise<void> {
  await db.insert(conversationAnalytics).values({
    id: generateId(),
    conversationId,
    userId,
    startedAt: new Date(),
    messageCount: 0,
    totalTokens: 0,
    category: 'unknown',
    outcome: 'in_progress'
  });
}

Tracking Message Exchange

typescript
export async function trackMessageExchange(
  conversationId: string,
  tokens: { input: number; output: number },
  responseTimeMs: number,
  flags: { hadFallback: boolean; hasCrisisIndicator: boolean }
): Promise<void> {
  await db
    .update(conversationAnalytics)
    .set({
      messageCount: sql`message_count + 1`,
      totalTokens: sql`total_tokens + ${tokens.input + tokens.output}`,
      inputTokens: sql`input_tokens + ${tokens.input}`,
      outputTokens: sql`output_tokens + ${tokens.output}`,
      avgResponseTime: sql`(avg_response_time * (message_count - 1) + ${responseTimeMs}) / message_count`,
      hadFallback: flags.hadFallback,
      ...(flags.hasCrisisIndicator && { outcome: 'crisis_escalated' })
    })
    .where(eq(conversationAnalytics.conversationId, conversationId));
}

Tracking Conversation End

typescript
export async function trackConversationEnd(
  conversationId: string,
  outcome: 'completed' | 'abandoned' | 'error'
): Promise<void> {
  await db
    .update(conversationAnalytics)
    .set({
      endedAt: new Date(),
      outcome
    })
    .where(eq(conversationAnalytics.conversationId, conversationId));
}

Category Detection (Metadata-Based)

Detect conversation categories WITHOUT reading content:

typescript
// Categories based on metadata flags from AI response
interface AIResponseMetadata {
  usedCopingStrategies: boolean;
  usedCrisisProtocol: boolean;
  usedCheckInSupport: boolean;
  usedGeneralChat: boolean;
  requestedClarification: boolean;
}

function deriveCategory(metadata: AIResponseMetadata): string {
  if (metadata.usedCrisisProtocol) return 'crisis_support';
  if (metadata.usedCopingStrategies) return 'coping_strategies';
  if (metadata.usedCheckInSupport) return 'checkin_support';
  if (metadata.requestedClarification) return 'clarification';
  return 'general_chat';
}

Dashboard Aggregations

Session Metrics

typescript
// Get aggregated session stats (HIPAA safe - no individual data)
async function getSessionStats(days: number = 30) {
  const since = subDays(new Date(), days);

  return db
    .select({
      totalSessions: count(),
      avgMessages: avg(conversationAnalytics.messageCount),
      avgDuration: avg(
        sql`JULIANDAY(ended_at) - JULIANDAY(started_at)) * 24 * 60`
      ),
      completionRate: sql`
        CAST(SUM(CASE WHEN outcome = 'completed' THEN 1 ELSE 0 END) AS FLOAT) /
        CAST(COUNT(*) AS FLOAT)
      `,
      crisisEscalations: sql`
        SUM(CASE WHEN outcome = 'crisis_escalated' THEN 1 ELSE 0 END)
      `
    })
    .from(conversationAnalytics)
    .where(gte(conversationAnalytics.startedAt, since));
}

Token Usage for Cost Tracking

typescript
async function getTokenUsage(days: number = 30) {
  const since = subDays(new Date(), days);

  const result = await db
    .select({
      totalTokens: sum(conversationAnalytics.totalTokens),
      inputTokens: sum(conversationAnalytics.inputTokens),
      outputTokens: sum(conversationAnalytics.outputTokens),
      avgTokensPerSession: avg(conversationAnalytics.totalTokens)
    })
    .from(conversationAnalytics)
    .where(gte(conversationAnalytics.startedAt, since));

  // Estimate cost (Claude pricing)
  const inputCost = (result.inputTokens / 1_000_000) * 3.00;  // $3/M input
  const outputCost = (result.outputTokens / 1_000_000) * 15.00; // $15/M output

  return {
    ...result,
    estimatedCost: inputCost + outputCost
  };
}

Category Breakdown

typescript
async function getCategoryBreakdown(days: number = 30) {
  const since = subDays(new Date(), days);

  return db
    .select({
      category: conversationAnalytics.category,
      count: count(),
      percentage: sql`
        CAST(COUNT(*) AS FLOAT) * 100.0 /
        (SELECT COUNT(*) FROM conversation_analytics WHERE started_at >= ${since})
      `
    })
    .from(conversationAnalytics)
    .where(gte(conversationAnalytics.startedAt, since))
    .groupBy(conversationAnalytics.category)
    .orderBy(desc(count()));
}

Alert Configuration

Set up alerts for concerning patterns:

typescript
interface AnalyticsAlert {
  type: 'crisis_spike' | 'error_spike' | 'abandonment_spike';
  threshold: number;
  windowHours: number;
  action: 'log' | 'email' | 'slack';
}

const alerts: AnalyticsAlert[] = [
  {
    type: 'crisis_spike',
    threshold: 5,  // 5+ crisis escalations
    windowHours: 24,
    action: 'email'
  },
  {
    type: 'error_spike',
    threshold: 10, // 10+ errors
    windowHours: 1,
    action: 'slack'
  },
  {
    type: 'abandonment_spike',
    threshold: 0.5, // 50%+ abandonment rate
    windowHours: 24,
    action: 'log'
  }
];

Database Schema

sql
CREATE TABLE conversation_analytics (
  id TEXT PRIMARY KEY,
  conversation_id TEXT NOT NULL,
  user_id TEXT NOT NULL,
  started_at TEXT NOT NULL,
  ended_at TEXT,
  message_count INTEGER DEFAULT 0,
  user_message_count INTEGER DEFAULT 0,
  ai_message_count INTEGER DEFAULT 0,
  total_tokens INTEGER DEFAULT 0,
  input_tokens INTEGER DEFAULT 0,
  output_tokens INTEGER DEFAULT 0,
  category TEXT DEFAULT 'unknown',
  outcome TEXT DEFAULT 'in_progress',
  avg_response_time REAL DEFAULT 0,
  had_fallback INTEGER DEFAULT 0,

  FOREIGN KEY (conversation_id) REFERENCES conversations(id),
  FOREIGN KEY (user_id) REFERENCES users(id)
);

CREATE INDEX idx_conv_analytics_started ON conversation_analytics(started_at);
CREATE INDEX idx_conv_analytics_user ON conversation_analytics(user_id);
CREATE INDEX idx_conv_analytics_outcome ON conversation_analytics(outcome);

Testing Analytics

typescript
describe('Conversation Analytics', () => {
  it('tracks session without PHI', async () => {
    const analytics = await trackConversationStart('conv-123', 'user-456');

    // Verify no PHI is stored
    expect(analytics).not.toHaveProperty('messageContent');
    expect(analytics).not.toHaveProperty('userQuery');

    // Verify metadata is stored
    expect(analytics.conversationId).toBe('conv-123');
    expect(analytics.messageCount).toBe(0);
  });

  it('calculates aggregates correctly', async () => {
    const stats = await getSessionStats(30);

    expect(stats.totalSessions).toBeGreaterThanOrEqual(0);
    expect(stats.completionRate).toBeBetween(0, 1);
  });
});

Resources