Add Observability

2. Register Metrics in Program.cs

// Register metrics
builder.Services.AddSingleton<NovaTuneMetrics>();

// Configure OpenTelemetry to export custom metrics
builder.Services.AddOpenTelemetry()
    .WithMetrics(metrics =>
    {
        metrics.AddMeter(NovaTuneMetrics.MeterName);
    });

3. Use Metrics in Services

public class TrackManagementService : ITrackManagementService
{
    private readonly NovaTuneMetrics _metrics;
    private readonly ILogger<TrackManagementService> _logger;

    public TrackManagementService(
        NovaTuneMetrics metrics,
        ILogger<TrackManagementService> logger)
    {
        _metrics = metrics;
        _logger = logger;
    }

    public async Task<PagedResult<TrackListItem>> ListTracksAsync(
        string userId,
        TrackListQuery query,
        CancellationToken ct)
    {
        var stopwatch = Stopwatch.StartNew();
        try
        {
            var result = await DoListTracksAsync(userId, query, ct);
            _metrics.RecordTrackListRequest("success");
            return result;
        }
        catch (Exception)
        {
            _metrics.RecordTrackListRequest("error");
            throw;
        }
        finally
        {
            _metrics.RecordTrackListDuration(stopwatch.Elapsed.TotalMilliseconds);
        }
    }
}

4. Configure Structured Logging

Location: Program.cs or service setup

using Serilog;
using Serilog.Events;
using Serilog.Formatting.Compact;

// Bootstrap logger (before builder)
Log.Logger = new LoggerConfiguration()
    .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
    .Enrich.FromLogContext()
    .WriteTo.Console(new RenderedCompactJsonFormatter())
    .CreateBootstrapLogger();

// Full configuration (in builder)
builder.Services.AddSerilog((services, configuration) => configuration
    .ReadFrom.Configuration(builder.Configuration)
    .ReadFrom.Services(services)
    .MinimumLevel.Information()
    .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
    .MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning)
    .MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning)
    .Enrich.FromLogContext()
    .Enrich.WithEnvironmentName()
    .Enrich.WithMachineName()
    .Enrich.WithProperty("Service", "NovaTune.ApiService")
    .WriteTo.Console(new RenderedCompactJsonFormatter()));

5. Implement Structured Logging

// Good: Structured with semantic properties
_logger.LogInformation(
    "Track {TrackId} soft-deleted for user {UserId}, scheduled deletion at {ScheduledAt}",
    trackId,
    userId,
    scheduledDeletionAt);

// Good: Using scopes for correlation
using (_logger.BeginScope(new Dictionary<string, object>
{
    ["CorrelationId"] = correlationId,
    ["TrackId"] = trackId
}))
{
    _logger.LogInformation("Starting track deletion process");
    // ... operations
    _logger.LogInformation("Track deletion completed successfully");
}

// Good: Different levels for different scenarios
_logger.LogDebug("Track list requested: {Search}, {Status}, {Limit}", search, status, limit);
_logger.LogInformation("Track {TrackId} updated by user {UserId}", trackId, userId);
_logger.LogWarning("Access denied: User {UserId} attempted to access track {TrackId} owned by {OwnerId}",
    userId, trackId, ownerId);
_logger.LogError(ex, "Failed to physically delete track {TrackId}", trackId);

6. Add Distributed Tracing

using System.Diagnostics;

public class TrackManagementService : ITrackManagementService
{
    private static readonly ActivitySource ActivitySource = new("NovaTune.TrackManagement");

    public async Task DeleteTrackAsync(string trackId, string userId, CancellationToken ct)
    {
        using var activity = ActivitySource.StartActivity("track.delete");
        activity?.SetTag("track.id", trackId);
        activity?.SetTag("user.id", userId);

        try
        {
            // Step 1: Update status
            using (var dbActivity = ActivitySource.StartActivity("db.update_status"))
            {
                await UpdateStatusAsync(trackId, ct);
            }

            // Step 2: Write outbox
            using (var outboxActivity = ActivitySource.StartActivity("outbox.write"))
            {
                await WriteOutboxAsync(trackId, ct);
            }

            // Step 3: Invalidate cache
            using (var cacheActivity = ActivitySource.StartActivity("cache.invalidate"))
            {
                await InvalidateCacheAsync(trackId, ct);
            }

            activity?.SetStatus(ActivityStatusCode.Ok);
        }
        catch (Exception ex)
        {
            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
            activity?.RecordException(ex);
            throw;
        }
    }
}

7. Register Tracing in ServiceDefaults

The ServiceDefaults project configures OpenTelemetry tracing:

// In AddServiceDefaults extension
builder.Services.AddOpenTelemetry()
    .WithTracing(tracing =>
    {
        tracing
            .AddSource("NovaTune.TrackManagement")
            .AddSource("NovaTune.Streaming")
            .AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddRavenDBInstrumentation();
    });

Logging Guidelines

Redaction (NF-4.5)

Never log sensitive data:

// BAD - Don't log object keys
_logger.LogInformation("Deleting object {ObjectKey}", objectKey);

// GOOD - Log identifiers only
_logger.LogInformation("Deleting storage objects for track {TrackId}", trackId);

Metrics Summary

Testing Observability

[Fact]
public void Metrics_Should_RecordTrackListRequest()
{
    // Arrange
    var meterFactory = new TestMeterFactory();
    var metrics = new NovaTuneMetrics(meterFactory);

    // Act
    metrics.RecordTrackListRequest("success");

    // Assert
    var measurements = meterFactory.GetMeasurements("track_list_requests_total");
    measurements.Should().ContainSingle()
        .Which.Value.Should().Be(1);
}