Real-World DevOps Scenarios #

Practical Integration Guide
Complete end-to-end scenarios combining Docker, Kubernetes, Terraform, Ansible, Git, and CI/CD

---

Table of Contents #

1. Scenario 1: Deploy 3-Tier Web Application
2. Scenario 2: Complete CI/CD Pipeline Setup
3. Scenario 3: Zero-Downtime Deployment
4. Scenario 4: Disaster Recovery Setup
5. Scenario 5: Multi-Region Deployment (planned)
6. Scenario 6: Microservices Architecture (planned)
7. Scenario 7: Secure Production Environment (planned)
8. Scenario 8: Database Migration with Zero Downtime (planned)

---

Scenario 1: Deploy 3-Tier Web Application #

🎯 Objective #

Deploy a complete 3-tier application (frontend, backend, database) using Docker, Kubernetes, Terraform, and Ansible.

📋 Architecture #

┌─────────────────────────────────────────┐
│         Load Balancer (Ingress)         │
└────────────────┬────────────────────────┘
                 │
    ┌────────────┴────────────┐
    │                         │
┌───▼────────┐         ┌──────▼──────┐
│  Frontend  │         │   Backend   │
│  (React)   │────────▶│   (Node.js) │
│  3 replicas│         │  3 replicas │
└────────────┘         └──────┬──────┘
                              │
                       ┌──────▼──────┐
                       │  PostgreSQL │
                       │  (StatefulSet)│
                       └─────────────┘

🔧 Step 1: Infrastructure with Terraform #

# terraform/main.tf
terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~> 3.0"
    }
  }
  
  backend "azurerm" {
    resource_group_name  = "rg-terraform-state"
    storage_account_name = "sttfstate"
    container_name       = "tfstate"
    key                  = "production.tfstate"
  }
}

provider "azurerm" {
  features {}
}

# Resource Group
resource "azurerm_resource_group" "main" {
  name     = "rg-webapp-prod"
  location = "East US"
  
  tags = {
    Environment = "Production"
    Project     = "WebApp"
    ManagedBy   = "Terraform"
  }
}

# Virtual Network
resource "azurerm_virtual_network" "main" {
  name                = "vnet-webapp-prod"
  address_space       = ["10.0.0.0/16"]
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
}

resource "azurerm_subnet" "aks" {
  name                 = "subnet-aks"
  resource_group_name  = azurerm_resource_group.main.name
  virtual_network_name = azurerm_virtual_network.main.name
  address_prefixes     = ["10.0.1.0/24"]
}

# AKS Cluster
resource "azurerm_kubernetes_cluster" "main" {
  name                = "aks-webapp-prod"
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  dns_prefix          = "webapp"
  kubernetes_version  = "1.27.3"
  
  default_node_pool {
    name                = "system"
    node_count          = 3
    vm_size             = "Standard_D2s_v3"
    vnet_subnet_id      = azurerm_subnet.aks.id
    enable_auto_scaling = true
    min_count           = 3
    max_count           = 10
  }
  
  identity {
    type = "SystemAssigned"
  }
  
  network_profile {
    network_plugin = "azure"
    network_policy = "azure"
  }
  
  tags = {
    Environment = "Production"
  }
}

# PostgreSQL Database
resource "azurerm_postgresql_flexible_server" "main" {
  name                   = "psql-webapp-prod"
  resource_group_name    = azurerm_resource_group.main.name
  location               = azurerm_resource_group.main.location
  version                = "15"
  administrator_login    = "dbadmin"
  administrator_password = var.db_password
  
  storage_mb = 32768
  sku_name   = "GP_Standard_D2s_v3"
  
  backup_retention_days = 7
  
  high_availability {
    mode = "ZoneRedundant"
  }
}

resource "azurerm_postgresql_flexible_server_database" "main" {
  name      = "webapp"
  server_id = azurerm_postgresql_flexible_server.main.id
  collation = "en_US.utf8"
  charset   = "utf8"
}

# Outputs
output "kube_config" {
  value     = azurerm_kubernetes_cluster.main.kube_config_raw
  sensitive = true
}

output "database_fqdn" {
  value = azurerm_postgresql_flexible_server.main.fqdn
}

🐳 Step 2: Containerize Applications #

Backend Dockerfile:

# backend/Dockerfile
FROM node:18-alpine AS builder

WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production

COPY . .
RUN npm run build

FROM node:18-alpine

WORKDIR /app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/node_modules ./node_modules
COPY package*.json ./

ENV NODE_ENV=production
EXPOSE 3000

USER node
CMD ["node", "dist/server.js"]

Frontend Dockerfile:

# frontend/Dockerfile
FROM node:18-alpine AS builder

WORKDIR /app
COPY package*.json ./
RUN npm ci

COPY . .
RUN npm run build

FROM nginx:alpine

COPY --from=builder /app/dist /usr/share/nginx/html
COPY nginx.conf /etc/nginx/nginx.conf

EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]

☸️ Step 3: Kubernetes Manifests #

Backend Deployment:

# k8s/backend-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend
  namespace: webapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
    spec:
      containers:
      - name: backend
        image: myregistry.azurecr.io/backend:latest
        ports:
        - containerPort: 3000
        env:
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: app-secrets
              key: database-url
        - name: NODE_ENV
          value: "production"
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 512Mi
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 5
          periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
  name: backend
  namespace: webapp
spec:
  selector:
    app: backend
  ports:
  - port: 80
    targetPort: 3000
  type: ClusterIP

Frontend Deployment:

# k8s/frontend-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frontend
  namespace: webapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: frontend
  template:
    metadata:
      labels:
        app: frontend
    spec:
      containers:
      - name: frontend
        image: myregistry.azurecr.io/frontend:latest
        ports:
        - containerPort: 80
        resources:
          requests:
            cpu: 50m
            memory: 64Mi
          limits:
            cpu: 200m
            memory: 256Mi
---
apiVersion: v1
kind: Service
metadata:
  name: frontend
  namespace: webapp
spec:
  selector:
    app: frontend
  ports:
  - port: 80
    targetPort: 80
  type: ClusterIP

Ingress:

# k8s/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: webapp-ingress
  namespace: webapp
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - webapp.example.com
    secretName: webapp-tls
  rules:
  - host: webapp.example.com
    http:
      paths:
      - path: /api
        pathType: Prefix
        backend:
          service:
            name: backend
            port:
              number: 80
      - path: /
        pathType: Prefix
        backend:
          service:
            name: frontend
            port:
              number: 80

🔄 Step 4: Ansible Configuration #

# ansible/playbook.yml
---
- name: Configure AKS cluster
  hosts: localhost
  gather_facts: no
  
  tasks:
    - name: Create namespace
      kubernetes.core.k8s:
        state: present
        definition:
          apiVersion: v1
          kind: Namespace
          metadata:
            name: webapp
    
    - name: Create database secret
      kubernetes.core.k8s:
        state: present
        definition:
          apiVersion: v1
          kind: Secret
          metadata:
            name: app-secrets
            namespace: webapp
          type: Opaque
          stringData:
            database-url: "{{ database_url }}"
    
    - name: Deploy backend
      kubernetes.core.k8s:
        state: present
        src: ../k8s/backend-deployment.yaml
    
    - name: Deploy frontend
      kubernetes.core.k8s:
        state: present
        src: ../k8s/frontend-deployment.yaml
    
    - name: Deploy ingress
      kubernetes.core.k8s:
        state: present
        src: ../k8s/ingress.yaml
    
    - name: Wait for deployments
      kubernetes.core.k8s_info:
        kind: Deployment
        namespace: webapp
        name: "{{ item }}"
        wait: yes
        wait_condition:
          type: Available
          status: "True"
        wait_timeout: 300
      loop:
        - backend
        - frontend

🚀 Step 5: CI/CD Pipeline #

# .github/workflows/deploy.yml
name: Deploy 3-Tier Application

on:
  push:
    branches: [ main ]

env:
  REGISTRY: myregistry.azurecr.io

jobs:
  build-backend:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Login to ACR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}
      
      - name: Build and push backend
        uses: docker/build-push-action@v5
        with:
          context: ./backend
          push: true
          tags: ${{ env.REGISTRY }}/backend:${{ github.sha }},${{ env.REGISTRY }}/backend:latest
  
  build-frontend:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Login to ACR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}
      
      - name: Build and push frontend
        uses: docker/build-push-action@v5
        with:
          context: ./frontend
          push: true
          tags: ${{ env.REGISTRY }}/frontend:${{ github.sha }},${{ env.REGISTRY }}/frontend:latest
  
  deploy:
    needs: [build-backend, build-frontend]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up kubectl
        uses: azure/setup-kubectl@v3
      
      - name: Azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      
      - name: Get AKS credentials
        run: |
          az aks get-credentials --resource-group rg-webapp-prod --name aks-webapp-prod
      
      - name: Deploy with Ansible
        run: |
          pip install ansible kubernetes
          ansible-playbook ansible/playbook.yml \
            -e "database_url=${{ secrets.DATABASE_URL }}"
      
      - name: Verify deployment
        run: |
          kubectl rollout status deployment/backend -n webapp
          kubectl rollout status deployment/frontend -n webapp

📝 Deployment Steps #

# 1. Provision infrastructure
cd terraform
terraform init
terraform plan -out=tfplan
terraform apply tfplan

# 2. Configure kubectl
az aks get-credentials --resource-group rg-webapp-prod --name aks-webapp-prod

# 3. Build and push containers
docker build -t myregistry.azurecr.io/backend:v1.0 ./backend
docker build -t myregistry.azurecr.io/frontend:v1.0 ./frontend
docker push myregistry.azurecr.io/backend:v1.0
docker push myregistry.azurecr.io/frontend:v1.0

# 4. Deploy with Ansible
cd ansible
ansible-playbook playbook.yml -e "database_url=postgresql://..."

# 5. Verify deployment
kubectl get pods -n webapp
kubectl get ingress -n webapp
curl https://webapp.example.com/health

---

Scenario 2: Complete CI/CD Pipeline Setup #

🎯 Objective #

Set up a complete CI/CD pipeline from code commit to production deployment with automated testing, security scanning, and rollback capabilities.

📋 Pipeline Flow #

┌──────────┐
│Git Push  │
└────┬─────┘
     │
┌────▼─────────────────────────────────────┐
│  GitHub Actions Triggered                │
└────┬─────────────────────────────────────┘
     │
     ├─────────────────┬─────────────────┐
     │                 │                 │
┌────▼────┐     ┌──────▼──────┐   ┌─────▼──────┐
│  Lint   │     │ Unit Tests  │   │SAST Scan   │
└────┬────┘     └──────┬──────┘   └─────┬──────┘
     │                 │                 │
     └────────┬────────┴─────────────────┘
              │
     ┌────────▼──────────┐
     │  Build Docker     │
     │  Image            │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Security Scan    │
     │  (Trivy)          │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Integration      │
     │  Tests            │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Deploy to        │
     │  Staging          │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  E2E Tests        │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Manual Approval  │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Deploy to        │
     │  Production       │
     │  (Blue-Green)     │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Smoke Tests      │
     └────────┬──────────┘
              │
     ┌────────▼──────────┐
     │  Notify Team      │
     └───────────────────┘

🔧 Complete Pipeline Configuration #

# .github/workflows/complete-pipeline.yml
name: Complete CI/CD Pipeline

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # Stage 1: Code Quality
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Run ESLint
        run: npm run lint
      
      - name: Run Prettier
        run: npm run format:check
  
  # Stage 2: Security Scanning (SAST)
  sast:
    runs-on: ubuntu-latest
    permissions:
      security-events: write
    steps:
      - uses: actions/checkout@v4
      
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: javascript
      
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2
      
      - name: Run Snyk scan
        uses: snyk/actions/node@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
  
  # Stage 3: Unit Tests
  test-unit:
    runs-on: ubuntu-latest
    needs: [lint]
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Run unit tests
        run: npm run test:unit -- --coverage
      
      - name: Upload coverage
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/lcov.info
          flags: unittests
  
  # Stage 4: Build Docker Image
  build:
    runs-on: ubuntu-latest
    needs: [test-unit, sast]
    if: github.event_name == 'push'
    permissions:
      contents: read
      packages: write
    outputs:
      image-tag: ${{ steps.meta.outputs.tags }}
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      
      - name: Log in to Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=ref,event=branch
            type=sha
      
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
  
  # Stage 5: Container Security Scan
  scan-image:
    runs-on: ubuntu-latest
    needs: [build]
    steps:
      - name: Run Trivy scan
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
          format: 'sarif'
          output: 'trivy-results.sarif'
      
      - name: Upload Trivy results
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'
  
  # Stage 6: Integration Tests
  test-integration:
    runs-on: ubuntu-latest
    needs: [build]
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_PASSWORD: testpass
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Run integration tests
        env:
          DATABASE_URL: postgresql://postgres:testpass@localhost:5432/testdb
          REDIS_URL: redis://localhost:6379
        run: npm run test:integration
  
  # Stage 7: Deploy to Staging
  deploy-staging:
    runs-on: ubuntu-latest
    needs: [scan-image, test-integration]
    if: github.ref == 'refs/heads/develop'
    environment:
      name: staging
      url: https://staging.example.com
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up kubectl
        uses: azure/setup-kubectl@v3
      
      - name: Azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      
      - name: Get AKS credentials
        run: az aks get-credentials --resource-group rg-staging --name aks-staging
      
      - name: Deploy to staging
        run: |
          kubectl set image deployment/myapp \
            myapp=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \
            -n staging
          kubectl rollout status deployment/myapp -n staging --timeout=5m
      
      - name: Run smoke tests
        run: |
          sleep 10
          curl -f https://staging.example.com/health || exit 1
  
  # Stage 8: E2E Tests
  test-e2e:
    runs-on: ubuntu-latest
    needs: [deploy-staging]
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
      
      - name: Install Playwright
        run: |
          npm ci
          npx playwright install --with-deps
      
      - name: Run E2E tests
        env:
          BASE_URL: https://staging.example.com
        run: npm run test:e2e
      
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: playwright-report
          path: playwright-report/
  
  # Stage 9: Deploy to Production
  deploy-production:
    runs-on: ubuntu-latest
    needs: [test-e2e]
    if: github.ref == 'refs/heads/main'
    environment:
      name: production
      url: https://example.com
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up kubectl
        uses: azure/setup-kubectl@v3
      
      - name: Azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      
      - name: Get AKS credentials
        run: az aks get-credentials --resource-group rg-production --name aks-production
      
      - name: Blue-Green Deployment
        run: |
          # Determine current active version
          ACTIVE=$(kubectl get svc myapp -n production -o jsonpath='{.spec.selector.version}')
          if [ "$ACTIVE" == "blue" ]; then
            INACTIVE="green"
          else
            INACTIVE="blue"
          fi
          
          echo "Deploying to $INACTIVE environment"
          
          # Deploy to inactive environment
          kubectl set image deployment/myapp-$INACTIVE \
            myapp=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \
            -n production
          
          # Wait for rollout
          kubectl rollout status deployment/myapp-$INACTIVE -n production --timeout=5m
          
          # Run smoke tests on inactive
          kubectl port-forward svc/myapp-$INACTIVE 8080:80 -n production &
          sleep 5
          curl -f http://localhost:8080/health || exit 1
          
          # Switch traffic
          kubectl patch svc myapp -n production -p \
            "{\"spec\":{\"selector\":{\"version\":\"$INACTIVE\"}}}"
          
          echo "Traffic switched to $INACTIVE"
      
      - name: Verify production deployment
        run: |
          sleep 10
          curl -f https://example.com/health
      
      - name: Notify team
        if: always()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "Production deployment ${{ job.status }}",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Deployment Status:* ${{ job.status }}\n*Version:* ${{ github.sha }}\n*Deployed by:* ${{ github.actor }}\n*URL:* https://example.com"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

📝 Setup Instructions #

# 1. Set up GitHub secrets
gh secret set AZURE_CREDENTIALS --body '{
  "clientId": "...",
  "clientSecret": "...",
  "subscriptionId": "...",
  "tenantId": "..."
}'
gh secret set SNYK_TOKEN --body "your-snyk-token"
gh secret set SLACK_WEBHOOK --body "your-slack-webhook-url"

# 2. Enable required GitHub features
# - Go to repository Settings > Security > Code scanning
# - Enable CodeQL analysis
# - Enable Dependabot alerts

# 3. Configure environments
# - Go to repository Settings > Environments
# - Create "staging" and "production" environments
# - Add required reviewers for production

# 4. Test the pipeline
git checkout -b feature/test-pipeline
git commit --allow-empty -m "Test pipeline"
git push origin feature/test-pipeline

---

Scenario 3: Zero-Downtime Deployment #

🎯 Objective #

Perform application updates without any downtime using Kubernetes rolling updates and health checks.

📋 Strategy #

# k8s/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  namespace: production
spec:
  replicas: 6
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2        # Allow 2 extra pods during update
      maxUnavailable: 0  # Never have unavailable pods
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
        version: v2.0
    spec:
      containers:
      - name: myapp
        image: myregistry.azurecr.io/myapp:v2.0
        ports:
        - containerPort: 8080
        
        # Readiness probe - when pod is ready to receive traffic
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
          successThreshold: 1
          failureThreshold: 3
        
        # Liveness probe - when pod should be restarted
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 15
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
        
        # Startup probe - for slow-starting applications
        startupProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 0
          periodSeconds: 10
          failureThreshold: 30  # 5 minutes to start
        
        # Graceful shutdown
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
        
        resources:
          requests:
            cpu: 200m
            memory: 256Mi
          limits:
            cpu: 500m
            memory: 512Mi
      
      # Ensure pods are distributed across nodes
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - myapp
              topologyKey: kubernetes.io/hostname
      
      # Graceful termination period
      terminationGracePeriodSeconds: 30

🚀 Deployment Script #

#!/bin/bash
# deploy-zero-downtime.sh

set -e

APP_NAME="myapp"
NAMESPACE="production"
NEW_VERSION="$1"

if [ -z "$NEW_VERSION" ]; then
  echo "Usage: $0 <version>"
  exit 1
fi

echo "Starting zero-downtime deployment of $APP_NAME to version $NEW_VERSION"

# 1. Update deployment image
kubectl set image deployment/$APP_NAME \
  $APP_NAME=myregistry.azurecr.io/$APP_NAME:$NEW_VERSION \
  -n $NAMESPACE \
  --record

# 2. Monitor rollout
echo "Monitoring rollout..."
kubectl rollout status deployment/$APP_NAME -n $NAMESPACE --timeout=10m

# 3. Verify all pods are ready
echo "Verifying pods..."
kubectl wait --for=condition=ready pod \
  -l app=$APP_NAME \
  -n $NAMESPACE \
  --timeout=5m

# 4. Run smoke tests
echo "Running smoke tests..."
POD=$(kubectl get pod -l app=$APP_NAME -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}')
kubectl exec $POD -n $NAMESPACE -- curl -f http://localhost:8080/health

# 5. Check service endpoints
ENDPOINTS=$(kubectl get endpoints $APP_NAME -n $NAMESPACE -o jsonpath='{.subsets[*].addresses[*].ip}' | wc -w)
echo "Service has $ENDPOINTS ready endpoints"

if [ "$ENDPOINTS" -lt 3 ]; then
  echo "ERROR: Not enough healthy endpoints!"
  kubectl rollout undo deployment/$APP_NAME -n $NAMESPACE
  exit 1
fi

echo "Deployment successful!"
echo "Version $NEW_VERSION is now serving traffic"

📊 Monitoring During Deployment #

# Watch pods during deployment
watch kubectl get pods -n production -l app=myapp

# Monitor events
kubectl get events -n production --watch

# Check rollout history
kubectl rollout history deployment/myapp -n production

# Rollback if needed
kubectl rollout undo deployment/myapp -n production

# Rollback to specific revision
kubectl rollout undo deployment/myapp -n production --to-revision=2

---

Scenario 4: Disaster Recovery Setup #

🎯 Objective #

Implement comprehensive backup and disaster recovery procedures for databases, configurations, and application state.

📋 Backup Strategy #

# k8s/backup-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: database-backup
  namespace: production
spec:
  schedule: "0 2 * * *"  # Daily at 2 AM
  successfulJobsHistoryLimit: 7
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: postgres:15
            env:
            - name: PGHOST
              value: "postgres.production.svc.cluster.local"
            - name: PGUSER
              valueFrom:
                secretKeyRef:
                  name: postgres-credentials
                  key: username
            - name: PGPASSWORD
              valueFrom:
                secretKeyRef:
                  name: postgres-credentials
                  key: password
            - name: AZURE_STORAGE_ACCOUNT
              valueFrom:
                secretKeyRef:
                  name: azure-credentials
                  key: storage-account
            - name: AZURE_STORAGE_KEY
              valueFrom:
                secretKeyRef:
                  name: azure-credentials
                  key: storage-key
            command:
            - /bin/bash
            - -c
            - |
              set -e
              
              # Create backup
              BACKUP_FILE="backup-$(date +%Y%m%d-%H%M%S).sql.gz"
              pg_dump -Fc myapp | gzip > /tmp/$BACKUP_FILE
              
              # Upload to Azure Blob Storage
              az storage blob upload \
                --account-name $AZURE_STORAGE_ACCOUNT \
                --account-key $AZURE_STORAGE_KEY \
                --container-name backups \
                --name database/$BACKUP_FILE \
                --file /tmp/$BACKUP_FILE
              
              # Verify backup
              az storage blob show \
                --account-name $AZURE_STORAGE_ACCOUNT \
                --account-key $AZURE_STORAGE_KEY \
                --container-name backups \
                --name database/$BACKUP_FILE
              
              echo "Backup completed: $BACKUP_FILE"
              
              # Cleanup old backups (keep last 30 days)
              CUTOFF_DATE=$(date -d '30 days ago' +%Y%m%d)
              az storage blob list \
                --account-name $AZURE_STORAGE_ACCOUNT \
                --account-key $AZURE_STORAGE_KEY \
                --container-name backups \
                --prefix database/ \
                --query "[?properties.creationTime < '$CUTOFF_DATE'].name" \
                -o tsv | \
              while read blob; do
                az storage blob delete \
                  --account-name $AZURE_STORAGE_ACCOUNT \
                  --account-key $AZURE_STORAGE_KEY \
                  --container-name backups \
                  --name "$blob"
                echo "Deleted old backup: $blob"
              done
          restartPolicy: OnFailure

🔄 Restore Procedure #

#!/bin/bash
# restore-database.sh

set -e

BACKUP_FILE="$1"
NAMESPACE="production"

if [ -z "$BACKUP_FILE" ]; then
  echo "Usage: $0 <backup-file>"
  echo "Example: $0 backup-20241201-020000.sql.gz"
  exit 1
fi

echo "WARNING: This will restore the database from backup: $BACKUP_FILE"
read -p "Are you sure? (yes/no): " CONFIRM

if [ "$CONFIRM" != "yes" ]; then
  echo "Restore cancelled"
  exit 0
fi

# 1. Scale down application
echo "Scaling down application..."
kubectl scale deployment/myapp --replicas=0 -n $NAMESPACE

# 2. Download backup
echo "Downloading backup..."
az storage blob download \
  --account-name $AZURE_STORAGE_ACCOUNT \
  --account-key $AZURE_STORAGE_KEY \
  --container-name backups \
  --name database/$BACKUP_FILE \
  --file /tmp/$BACKUP_FILE

# 3. Restore database
echo "Restoring database..."
gunzip < /tmp/$BACKUP_FILE | \
kubectl exec -i postgres-0 -n $NAMESPACE -- \
  psql -U postgres -d myapp

# 4. Verify restore
echo "Verifying restore..."
kubectl exec postgres-0 -n $NAMESPACE -- \
  psql -U postgres -d myapp -c "SELECT COUNT(*) FROM users;"

# 5. Scale up application
echo "Scaling up application..."
kubectl scale deployment/myapp --replicas=3 -n $NAMESPACE

# 6. Wait for pods to be ready
kubectl wait --for=condition=ready pod \
  -l app=myapp \
  -n $NAMESPACE \
  --timeout=5m

echo "Restore completed successfully!"

📦 Configuration Backup #

#!/bin/bash
# backup-k8s-config.sh

NAMESPACE="production"
BACKUP_DIR="k8s-backup-$(date +%Y%m%d)"

mkdir -p $BACKUP_DIR

# Backup all resources
kubectl get all -n $NAMESPACE -o yaml > $BACKUP_DIR/all-resources.yaml

# Backup specific resources
for resource in configmap secret deployment service ingress pvc; do
  kubectl get $resource -n $NAMESPACE -o yaml > $BACKUP_DIR/$resource.yaml
done

# Backup RBAC
kubectl get rolebinding,role -n $NAMESPACE -o yaml > $BACKUP_DIR/rbac.yaml

# Create archive
tar -czf $BACKUP_DIR.tar.gz $BACKUP_DIR

# Upload to storage
az storage blob upload \
  --account-name $AZURE_STORAGE_ACCOUNT \
  --account-key $AZURE_STORAGE_KEY \
  --container-name backups \
  --name k8s-config/$BACKUP_DIR.tar.gz \
  --file $BACKUP_DIR.tar.gz

echo "Kubernetes configuration backed up to: $BACKUP_DIR.tar.gz"

---

🎓 Real-World Scenarios Guide - Part 1 Complete

This guide demonstrates practical integration of all master class topics. Continue to Part 2 for Scenarios 5-8 covering multi-region deployments, microservices, security, and database migrations.