Advanced SRE Patterns

# Example: ML-based anomaly detection
 
from sklearn.ensemble import IsolationForest
import numpy as np
 
def detect_anomalies(time_series_data, contamination=0.01):
    """
    Use Isolation Forest to detect anomalous metrics
    
    Advantages over threshold-based:
    - Learns normal patterns (handles seasonality)
    - Adapts to changing baselines
    - Detects subtle anomalies
    """
    
    X = np.array(time_series_data).reshape(-1, 1)
    
    # Train model on historical data
    iso_forest = IsolationForest(
        contamination=contamination,  # ~1% of points are anomalies
        random_state=42
    )
    
    # Detect anomalies
    predictions = iso_forest.predict(X)
    
    # -1 = anomaly, 1 = normal
    anomaly_indices = np.where(predictions == -1)[0]
    
    return anomaly_indices
 
# Usage
cpu_metrics = [45, 48, 46, 47, 45, 75, 76, 74, 45, 46]  # Spike in middle
anomalies = detect_anomalies(cpu_metrics)
print(f"Anomalous points: {anomalies}")  # [5, 6, 7]

# Predict when capacity will be exceeded
 
from statsmodels.tsa.holtwinters import ExponentialSmoothing
 
def forecast_capacity(historical_usage, periods_ahead=30, capacity_limit=85):
    """Forecast when usage will hit capacity"""
    
    # Fit exponential smoothing model
    model = ExponentialSmoothing(
        historical_usage,
        seasonal_periods=7,  # Weekly seasonality
        trend='add',
        seasonal='add'
    )
    fit = model.fit()
    
    # Forecast ahead
    forecast = fit.forecast(steps=periods_ahead)
    
    # Find when it exceeds capacity
    days_until_capacity = None
    for day, value in enumerate(forecast):
        if value > capacity_limit:
            days_until_capacity = day
            break
    
    return {
        'forecast': forecast,
        'days_until_capacity': days_until_capacity,
        'action': 'Scale now' if days_until_capacity and days_until_capacity is less than 14 else 'Monitor'
    }

# CP (Consistency + Partition tolerance)
- Strong consistency across regions
- May not be available (wait for confirmation)
- Example: Financial transactions
 
# AP (Availability + Partition tolerance)
- Always available
- May have eventual consistency
- Example: Social media likes/comments
 
SRE Decision:
- For critical operations: Sacrifice some availability for consistency
- For UX features: Sacrifice some consistency for availability
- Monitor and test in chaos scenarios

// Example: Multi-region database
 
// Approach 1: Leader-Follower (Master-Slave)
// Master in Region A → Replica in Region B (one-way replication)
// Problem: Region B can't take writes (must failover)
 
// Approach 2: Multi-Master (Active-Active)
// Region A and Region B both accept writes
// Problem: Potential conflicts, need conflict resolution
 
@Entity
public class ConflictResolution {
    // Last-Writer-Wins: Latest timestamp wins
    @Column(columnDefinition = "timestamp DEFAULT CURRENT_TIMESTAMP")
    private LocalDateTime lastUpdated;
    
    // LWW conflict resolution
    public void mergeUpdates(ConflictResolution remote) {
        if (remote.lastUpdated.isAfter(this.lastUpdated)) {
            this.data = remote.data;
            this.lastUpdated = remote.lastUpdated;
        }
    }
}

Tools for chaos:
- Gremlin: Commercial chaos platform
- Chaos Mesh: Open source Kubernetes chaos
- Locust: Load testing and chaos
- Boundary: Network chaos
- Custom scripts: Specific to your systems
 
What to test:
- Server failure (terminate process/instance)
- Network latency (slow down or delay)
- Packet loss (drop % of traffic)
- Database failure (make queries timeout)
- Dependency failure (mock service returns errors)

# Instead of text logs, use structured logs
 
# ❌ Bad (text log)
logger.info("Request from 192.168.1.1 completed in 150ms with status 200")
 
# ✅ Good (structured log)
logger.info("request_complete", {
    'timestamp': '2024-01-15T09:30:00Z',
    'client_ip': '192.168.1.1',
    'service': 'payment-api',
    'endpoint': '/payments',
    'method': 'POST',
    'duration_ms': 150,
    'status_code': 200,
    'user_id': 'user-12345',
    'transaction_id': 'txn-98765',
    'trace_id': 'trace-abc123',
    'severity': 'info'
})

# Advanced alerting with context
 
Alert: High Error Rate
  Service: payment-api
  Severity: Critical
  
  Context:
    - Deployment: v2.3.1 deployed 15 min ago
    - Recent changes: Added new payment processor
    - Database: Replication lag is normal
    - Dependencies: All healthy
    
  Root cause detection:
    - 90% of errors from new payment processor
    - Processor failing with auth timeouts
    - Auth service healthy (not cause)
    - New processor has wrong credentials
    
  Recommended action:
    - Immediate: Roll back to v2.3.0
    - Quick: Verify credentials for new processor
    - Follow-up: Add pre-deployment validation

# Automatically (without code changes):
- Request tracing across services
- Latency metrics per service pair
- mTLS encryption between services
- Retry logic and circuit breakers
- Rate limiting and load balancing
 
# Example metrics automatically provided:
requests[
  source_service="payment-api",
  dest_service="user-service",
  status="success",
  latency="method_d"
]
 
# Immediately get insights:
- traffic flow between services
- error rates
- latency per path
- Can identify bottlenecks automatically

# Track service dependencies
 
class ServiceDependencyGraph:
    dependencies = {
        'payments': ['user-service', 'auth', 'database'],
        'orders': ['user-service', 'inventory', 'database'],
        'inventory': ['warehouse', 'database'],
    }
    
    def critical_services(self):
        """Find services that break everything if they fail"""
        dependency_count = {}
        for service, deps in self.dependencies.items():
            for dep in deps:
                dependency_count[dep] = dependency_count.get(dep, 0) + 1
        
        # Services depended on by many others
        return sorted(dependency_count.items(), key=lambda x: x[1], reverse=True)
    
    # Probably: auth and user-service are most critical
 
# Use this to prioritize:
# - Which services need highest SLO?
# - Which services to focus chaos testing on?
# - Where to invest reliability efforts?

# Investment in reliability vs. benefit
 
class ReliabilityROI:
    def __init__(self, service):
        self.service = service
        
    def calculate_optimal_reliability(self):
        # Cost of infrastructure for N nines
        cost_99_0 = 10_000    # $10k/month for 99%
        cost_99_9 = 30_000    # $30k/month for 99.9%
        cost_99_99 = 100_000  # $100k/month for 99.99%
        
        # Cost of downtime
        downtime_cost_per_min = 1_000  # $1000 per minute
        
        # Downtime at each reliability level (annual)
        downtime_99_0 = 365 * 24 * 60 * 0.01    # 5,256 minutes
        downtime_99_9 = 365 * 24 * 60 * 0.001   # 526 minutes
        downtime_99_99 = 365 * 24 * 60 * 0.0001 # 53 minutes
        
        # Total cost per year
        total_99_0 = cost_99_0 * 12 + downtime_99_0 * downtime_cost_per_min
        total_99_9 = cost_99_9 * 12 + downtime_99_9 * downtime_cost_per_min
        total_99_99 = cost_99_99 * 12 + downtime_99_99 * downtime_cost_per_min
        
        return {
            '99%': total_99_0,
            '99.9%': total_99_9,
            '99.99%': total_99_99,
            'optimal': min([
                ('99%', total_99_0),
                ('99.9%', total_99_9),
                ('99.99%', total_99_99)
            ], key=lambda x: x[1])
        }
 
# Results might show 99.9% is optimal for cost + benefit

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: payment-api
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: payment-api
  service:
    port: 8080
  analysis:
    interval: 1m
    threshold: 5
    metrics:
    - name: error-rate
      thresholdRange:
        max: 0.5  # Max 0.5% errors
    - name: latency
      thresholdRange:
        max: 100  # Max 100ms latency
    webhooks:
    - name: smoke-tests
      url: http://flagger-loadtester/
  skipAnalysis: false
  progressDeadlineSeconds: 60
  # Canary stages
  stages:
  - weight: 5     # 5% of traffic
    maxWeight: 5
    stepWeight: 1
    metrics:
    - name: error-rate
      interval: 1m
  - weight: 25    # 25% of traffic
    metrics:
    - name: error-rate
    - name: latency
  - weight: 50    # 50% of traffic
    metrics:
    - name: error-rate
    - name: latency
  # Then automatic 100%