observability-logging-baseline

// Correct: Message template with named placeholders
_logger.LogInformation("User {UserId} logged in from {IpAddress}", userId, ipAddress);

// Incorrect: String interpolation loses structure
_logger.LogInformation($"User {userId} logged in from {ipAddress}");

// Middleware to propagate correlation ID
public class CorrelationIdMiddleware
{
    public async Task InvokeAsync(HttpContext context)
    {
        var correlationId = context.Request.Headers["X-Correlation-ID"].FirstOrDefault()
            ?? Guid.NewGuid().ToString();

        context.Items["CorrelationId"] = correlationId;
        context.Response.Headers["X-Correlation-ID"] = correlationId;

        using (_logger.BeginScope(new Dictionary<string, object>
        {
            ["CorrelationId"] = correlationId
        }))
        {
            await _next(context);
        }
    }
}

// Mask sensitive fields
_logger.LogInformation("Payment processed for card ending {CardLast4}",
    cardNumber.Substring(cardNumber.Length - 4));

// Use redaction attributes
[LogPropertyIgnore]
public string Password { get; set; }

builder.Services.AddOpenTelemetry()
    .WithMetrics(metrics =>
    {
        metrics
            .AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddRuntimeInstrumentation()
            .AddOtlpExporter();
    });

# Format: <namespace>.<metric_name>_<unit>
http.server.request.duration   (histogram, seconds)
http.server.active_requests    (gauge)
db.client.connections.usage    (gauge)

traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
tracestate: vendor1=value1,vendor2=value2

builder.Services.AddOpenTelemetry()
    .WithTracing(tracing =>
    {
        tracing
            .AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddSqlClientInstrumentation()
            .AddOtlpExporter();
    });

// Probability-based sampling
tracing.SetSampler(new TraceIdRatioBasedSampler(0.1)); // 10% sampling

// JSON output for container log aggregation
builder.Host.UseSerilog((context, config) =>
{
    config
        .WriteTo.Console(new JsonFormatter())
        .Enrich.WithProperty("ServiceName", "my-api");
});

.ConfigureResource(resource =>
{
    resource
        .AddService("my-api")
        .AddAttributes(new Dictionary<string, object>
        {
            ["k8s.namespace.name"] = Environment.GetEnvironmentVariable("K8S_NAMESPACE"),
            ["k8s.pod.name"] = Environment.GetEnvironmentVariable("K8S_POD_NAME")
        });
});

.AddOtlpExporter(options =>
{
    options.Endpoint = new Uri("http://otel-collector:4317");
    options.Protocol = OtlpExportProtocol.Grpc;
});

// Jaeger, Zipkin, or vendor-specific exporters
// Only use when OTLP collector is not available
.AddJaegerExporter()

{
  "timestamp": "2024-01-15T10:30:45.123Z",
  "level": "Information",
  "message": "User logged in successfully",
  "properties": {
    "UserId": "usr_12345",
    "IpAddress": "192.168.1.100",
    "CorrelationId": "abc-123-def-456",
    "ServiceName": "auth-api"
  }
}

# Check log output format
curl -s http://localhost:5000/api/health | jq .
# Then check application logs for structured output

# Verify correlation ID propagation
curl -H "X-Correlation-ID: test-123" http://localhost:5000/api/users/1
# Check logs contain "CorrelationId": "test-123"

http_server_request_duration_seconds_bucket{method="GET",route="/api/users/{id}",status="200",le="0.1"}
http_server_request_duration_seconds_count{method="GET",route="/api/users/{id}",status="200"}
http_server_active_requests{method="GET"}

# Check metrics endpoint
curl -s http://localhost:5000/metrics | grep http_server

# Verify histogram buckets exist
curl -s http://localhost:5000/metrics | grep "_bucket"

# Check for high-cardinality labels (should NOT see user IDs, request IDs, etc.)
curl -s http://localhost:5000/metrics | grep -E "user_id|request_id"
# Should return nothing

traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
tracestate: myvendor=value

# Verify trace context propagation
# Make request with traceparent header
curl -H "traceparent: 00-12345678901234567890123456789012-1234567890123456-01" \
     -v http://localhost:5000/api/users/1 2>&1 | grep -i traceparent

# Check response includes traceparent
# Verify downstream services receive same trace ID

# Check OTLP exporter connectivity
curl -s http://localhost:4317/health  # gRPC health check
curl -s http://localhost:4318/health  # HTTP health check

#!/bin/bash
# validate-observability.sh

SERVICE_URL="${1:-http://localhost:5000}"
ERRORS=0

echo "=== Observability Validation ==="
echo "Service: $SERVICE_URL"
echo ""

# Test 1: Metrics endpoint
echo "1. Checking metrics endpoint..."
if curl -sf "$SERVICE_URL/metrics" > /dev/null; then
  echo "   ✓ Metrics endpoint accessible"

  # Check for RED metrics
  if curl -sf "$SERVICE_URL/metrics" | grep -q "http_server_request_duration"; then
    echo "   ✓ Duration metric present"
  else
    echo "   ✗ Duration metric missing"
    ((ERRORS++))
  fi
else
  echo "   ✗ Metrics endpoint not accessible"
  ((ERRORS++))
fi

# Test 2: Correlation ID propagation
echo ""
echo "2. Checking correlation ID propagation..."
CORRELATION_ID="test-$(date +%s)"
RESPONSE=$(curl -sf -H "X-Correlation-ID: $CORRELATION_ID" "$SERVICE_URL/api/health" -i 2>&1)
if echo "$RESPONSE" | grep -qi "x-correlation-id"; then
  echo "   ✓ Correlation ID returned in response"
else
  echo "   ✗ Correlation ID not propagated"
  ((ERRORS++))
fi

# Test 3: Trace context
echo ""
echo "3. Checking trace context propagation..."
TRACE_ID="12345678901234567890123456789012"
RESPONSE=$(curl -sf -H "traceparent: 00-$TRACE_ID-1234567890123456-01" \
                "$SERVICE_URL/api/health" -i 2>&1)
if echo "$RESPONSE" | grep -qi "traceparent"; then
  echo "   ✓ Trace context propagated"
else
  echo "   ⚠ Trace context not in response headers (may be logged internally)"
fi

# Summary
echo ""
echo "=== Summary ==="
if [ $ERRORS -eq 0 ]; then
  echo "✓ All validations passed"
  exit 0
else
  echo "✗ $ERRORS validation(s) failed"
  exit 1
fi

# Observability Validation Evidence

**Service**: [service-name]
**Date**: YYYY-MM-DD
**Validator**: [name]

## Logging

- [x] Structured logging configured
- [x] Correlation ID middleware active
- [x] Log level: Information (production)

**Sample log entry:**
[paste actual log entry here]

## Metrics

- [x] Metrics endpoint: /metrics
- [x] RED metrics present
- [x] No high-cardinality labels

**Metrics output:**

```text
[paste relevant metrics here]
```

## Tracing

- [x] OpenTelemetry SDK configured
- [x] W3C Trace Context propagation
- [x] Sampling: 10% (production)

**Trace verification:**
[paste trace ID and verification steps]