OpenTelemetry Go Instrumentation

Expert guidance for instrumenting Go applications with OpenTelemetry using manual instrumentation and SDK configuration.

Quick Start

Install Dependencies

bash

go get go.opentelemetry.io/otel \
  go.opentelemetry.io/otel/sdk \
  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc \
  go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc

Initialize OpenTelemetry

package main

import (
    "context"
    "log"
    
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
)

func initTracer() (*sdktrace.TracerProvider, error) {
    exporter, err := otlptracegrpc.New(context.Background(),
        otlptracegrpc.WithEndpoint("localhost:4317"),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        return nil, err
    }
    
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName("my-service"),
            semconv.ServiceVersion("1.0.0"),
        )),
    )
    
    otel.SetTracerProvider(tp)
    return tp, nil
}

Core Instrumentation Patterns

Creating Spans

import (
    "context"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/codes"
)

func processRequest(ctx context.Context, userID string) error {
    tracer := otel.Tracer("my-service")
    ctx, span := tracer.Start(ctx, "process-request")
    defer span.End()
    
    span.SetAttributes(attribute.String("user.id", userID))
    
    if err := doWork(ctx); err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return err
    }
    
    span.SetStatus(codes.Ok, "Success")
    return nil
}

Context Propagation

Always pass context.Context through your call chain:

func parentFunction(ctx context.Context) {
    ctx, span := tracer.Start(ctx, "parent")
    defer span.End()
    
    // Context automatically propagates to child
    childFunction(ctx)
}

func childFunction(ctx context.Context) {
    // Child span inherits parent context
    _, span := tracer.Start(ctx, "child")
    defer span.End()
    
    // Do work
}

HTTP Instrumentation

HTTP Server

import (
    "net/http"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func main() {
    // Wrap handler with otelhttp middleware
    handler := http.HandlerFunc(handleRequest)
    wrappedHandler := otelhttp.NewHandler(handler, "my-service")
    
    http.Handle("/", wrappedHandler)
    http.ListenAndServe(":8080", nil)
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
    // Span automatically created by middleware
    ctx := r.Context()
    
    // Add custom attributes
    span := trace.SpanFromContext(ctx)
    span.SetAttributes(attribute.String("custom.key", "value"))
    
    w.Write([]byte("Hello, World!"))
}

HTTP Client

import (
    "net/http"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func makeRequest(ctx context.Context, url string) error {
    client := http.Client{
        Transport: otelhttp.NewTransport(http.DefaultTransport),
    }
    
    req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
    if err != nil {
        return err
    }
    
    resp, err := client.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    
    return nil
}

gRPC Instrumentation

gRPC Server

import (
    "google.golang.org/grpc"
    "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
)

func main() {
    server := grpc.NewServer(
        grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
        grpc.StreamInterceptor(otelgrpc.StreamServerInterceptor()),
    )
    
    // Register services
    pb.RegisterMyServiceServer(server, &myService{})
}

gRPC Client

func createClient() (*grpc.ClientConn, error) {
    conn, err := grpc.Dial("localhost:50051",
        grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
        grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
    )
    return conn, err
}

Metrics

Creating Metrics

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/metric"
)

func setupMetrics() {
    meter := otel.Meter("my-service")
    
    // Counter
    counter, _ := meter.Int64Counter(
        "http.requests",
        metric.WithDescription("Total HTTP requests"),
    )
    counter.Add(ctx, 1, metric.WithAttributes(
        attribute.String("http.method", "GET"),
    ))
    
    // Histogram
    histogram, _ := meter.Float64Histogram(
        "http.request.duration",
        metric.WithUnit("ms"),
    )
    histogram.Record(ctx, 150.5, metric.WithAttributes(
        attribute.String("http.method", "GET"),
    ))
    
    // Async Gauge
    gauge, _ := meter.Int64ObservableGauge(
        "process.memory.usage",
        metric.WithUnit("bytes"),
    )
    meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
        var m runtime.MemStats
        runtime.ReadMemStats(&m)
        o.ObserveInt64(gauge, int64(m.Alloc))
        return nil
    }, gauge)
}

Database Instrumentation

SQL Database

import (
    "database/sql"
    "go.opentelemetry.io/contrib/instrumentation/database/sql/otelsql"
    semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
)

func openDB() (*sql.DB, error) {
    db, err := otelsql.Open("postgres", "connection-string",
        otelsql.WithAttributes(semconv.DBSystemPostgreSQL),
    )
    if err != nil {
        return nil, err
    }
    
    // Register stats to get database metrics
    otelsql.RegisterDBStatsMetrics(db, otelsql.WithAttributes(
        semconv.DBSystemPostgreSQL,
    ))
    
    return db, nil
}

Goroutine Context Propagation

func processInGoroutine(ctx context.Context) {
    ctx, span := tracer.Start(ctx, "parent-operation")
    defer span.End()
    
    // Pass context to goroutine
    go func(ctx context.Context) {
        _, childSpan := tracer.Start(ctx, "child-operation")
        defer childSpan.End()
        
        // Do work
    }(ctx)
}

Sampling

Probability Sampler

import (
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

tp := sdktrace.NewTracerProvider(
    sdktrace.WithSampler(sdktrace.ParentBased(
        sdktrace.TraceIDRatioBased(0.1), // Sample 10%
    )),
)

Best Practices

•Always pass context - Use context.Context for all function calls
•Defer span.End() - Always close spans with defer span.End()
•Record errors - Use span.RecordError(err) and span.SetStatus()
•Resource attributes - Set service.name and other identifying attributes
•Use middleware - Leverage otelhttp and otelgrpc for automatic instrumentation
•Context propagation - Pass context to goroutines for proper tracing
•Batch processing - Use WithBatcher() for better performance
•Semantic conventions - Follow OpenTelemetry semantic conventions

Advanced Topics

For detailed examples covering advanced HTTP patterns, database instrumentation, microservices, and complex scenarios, see references/examples.md.

Troubleshooting

Spans not appearing

•Verify exporter endpoint is accessible
•Check span.End() is called
•Ensure TracerProvider is set globally

Context not propagating

•Always pass context.Context as first parameter
•Check middleware is properly configured
•Verify context is passed to goroutines

Performance issues

•Use WithBatcher() instead of synchronous export
•Adjust batch size and timeout
•Implement appropriate sampling

Key Dependencies

•go.opentelemetry.io/otel - Core API
•go.opentelemetry.io/otel/sdk - SDK implementation
•go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc - OTLP trace exporter
•go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc - OTLP metric exporter
•go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp - HTTP instrumentation
•go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc - gRPC instrumentation

opentelemetry-go