Go Observability Stack

1. Logger Setup

// internal/infrastructure/logging/logger.go
package logging

import (
    "os"
    "github.com/sirupsen/logrus"
    "go.elastic.co/ecslogrus"
)

func NewLogger() *logrus.Logger {
    logger := logrus.New()
    logger.SetFormatter(&ecslogrus.Formatter{})
    logger.SetOutput(os.Stdout)
    logger.SetLevel(logrus.InfoLevel)
    return logger
}

2. OTEL Hook for Trace Correlation

// internal/infrastructure/logging/otel_hook.go
package logging

import (
    "github.com/sirupsen/logrus"
    "go.opentelemetry.io/otel/trace"
)

type OTELHook struct{}

func NewOTELHook() *OTELHook {
    return &OTELHook{}
}

func (hook *OTELHook) Levels() []logrus.Level {
    return logrus.AllLevels
}

func (hook *OTELHook) Fire(entry *logrus.Entry) error {
    ctx := entry.Context
    if ctx == nil {
        return nil
    }

    spanCtx := trace.SpanFromContext(ctx).SpanContext()
    if !spanCtx.IsValid() {
        return nil
    }

    // Add ECS-compliant trace fields
    entry.Data["trace.id"] = spanCtx.TraceID().String()
    entry.Data["span.id"] = spanCtx.SpanID().String()

    if spanCtx.IsSampled() {
        entry.Data["trace.flags"] = "01"
    }

    return nil
}

3. OpenTelemetry Initialization

// internal/infrastructure/observability/otel.go
package observability

import (
    "context"
    "fmt"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    sdkmetric "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

type OTELShutdown func(context.Context) error

func InitOTEL(ctx context.Context, serviceName, serviceVersion, collectorEndpoint string) (OTELShutdown, error) {
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName(serviceName),
            semconv.ServiceVersion(serviceVersion),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Use grpc.Dial, NOT grpc.NewClient
    conn, err := grpc.Dial(collectorEndpoint,
        grpc.WithTransportCredentials(insecure.NewCredentials()),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create gRPC connection: %w", err)
    }

    traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    traceProvider := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(traceExporter),
        sdktrace.WithResource(res),
    )
    otel.SetTracerProvider(traceProvider)

    metricExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create metric exporter: %w", err)
    }

    metricProvider := sdkmetric.NewMeterProvider(
        sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)),
        sdkmetric.WithResource(res),
    )
    otel.SetMeterProvider(metricProvider)

    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return func(ctx context.Context) error {
        ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
        defer cancel()

        if err := traceProvider.Shutdown(ctx); err != nil {
            return err
        }
        if err := metricProvider.Shutdown(ctx); err != nil {
            return err
        }
        return conn.Close()
    }, nil
}

4. Gin Server with OTEL Middleware

// internal/infrastructure/server/server.go
package server

import (
    "github.com/gin-gonic/gin"
    "github.com/sirupsen/logrus"
    "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
)

func SetupRouter(handler *Handler, serviceName string, logger *logrus.Logger) *gin.Engine {
    gin.SetMode(gin.ReleaseMode)
    router := gin.New()

    router.Use(gin.Recovery())
    router.Use(LoggingMiddleware(logger))
    router.Use(otelgin.Middleware(serviceName))  // OTEL tracing

    router.GET("/health", handler.Health)
    return router
}

func LoggingMiddleware(logger *logrus.Logger) gin.HandlerFunc {
    return func(c *gin.Context) {
        ctx := c.Request.Context()
        c.Next()

        logger.WithContext(ctx).WithFields(logrus.Fields{
            "http.request.method":       c.Request.Method,
            "url.path":                  c.Request.URL.Path,
            "http.response.status_code": c.Writer.Status(),
            "client.address":            c.ClientIP(),
        }).Info("HTTP request completed")
    }
}

5. Wire in Main

// cmd/main.go
func main() {
    logger := logging.NewLogger()
    logger.AddHook(logging.NewOTELHook())  // Enable trace correlation

    ctx := context.Background()
    otelShutdown, err := observability.InitOTEL(ctx, "my-service", "1.0.0", "localhost:4317")
    if err != nil {
        logger.Fatalf("OTEL init failed: %v", err)
    }
    defer otelShutdown(ctx)

    router := server.SetupRouter(handler, "my-service", logger)
    router.Run(":8080")
}

6. OTEL Collector Config

# .config/otel-collector.yml