Lesson 4 of 14 ~100 min
Course progress
0%

Docker & Kubernetes Test Deployment

Deploy and scale Selenium tests using Docker containers and Kubernetes orchestration

1 Docker & Kubernetes Test Deployment

Docker & Kubernetes Test Deployment

Master containerized test deployment and orchestration for enterprise-scale test automation.

Docker Basics for Test Automation

Dockerfile for Test Container

# Dockerfile
FROM maven:3.9-eclipse-temurin-17

# Install Chrome
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
RUN apt-get update && apt-get install -y google-chrome-stable

# Install ChromeDriver
RUN CHROME_VERSION=$(google-chrome --version | awk '{print $3}' | cut -d'.' -f1) && \
    wget -q "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_VERSION}" -O /tmp/version && \
    DRIVER_VERSION=$(cat /tmp/version) && \
    wget -q "https://chromedriver.storage.googleapis.com/${DRIVER_VERSION}/chromedriver_linux64.zip" && \
    unzip chromedriver_linux64.zip -d /usr/local/bin && \
    chmod +x /usr/local/bin/chromedriver

# Set working directory
WORKDIR /app

# Copy project files
COPY pom.xml .
COPY src ./src

# Run tests
CMD ["mvn", "clean", "test", "-Dheadless=true"]

Docker Compose for Local Testing

# docker-compose.yml
version: '3.8'

services:
  selenium-hub:
    image: selenium/hub:4.15.0
    ports:
      - "4444:4444"
    environment:
      - SE_SESSION_REQUEST_TIMEOUT=300
      - SE_SESSION_RETRY_INTERVAL=5
      - SE_NODE_MAX_SESSIONS=5
    networks:
      - selenium-grid

  chrome-node:
    image: selenium/node-chrome:4.15.0
    depends_on:
      - selenium-hub
    environment:
      - SE_EVENT_BUS_HOST=selenium-hub
      - SE_EVENT_BUS_PUBLISH_PORT=4442
      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
      - SE_NODE_MAX_SESSIONS=3
    volumes:
      - /dev/shm:/dev/shm
    networks:
      - selenium-grid
    deploy:
      replicas: 3

  firefox-node:
    image: selenium/node-firefox:4.15.0
    depends_on:
      - selenium-hub
    environment:
      - SE_EVENT_BUS_HOST=selenium-hub
      - SE_EVENT_BUS_PUBLISH_PORT=4442
      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
      - SE_NODE_MAX_SESSIONS=3
    volumes:
      - /dev/shm:/dev/shm
    networks:
      - selenium-grid
    deploy:
      replicas: 2

  test-runner:
    build: .
    depends_on:
      - selenium-hub
    environment:
      - SELENIUM_GRID_URL=http://selenium-hub:4444
      - TEST_ENV=docker
    networks:
      - selenium-grid

networks:
  selenium-grid:
    driver: bridge

Kubernetes Deployment

Selenium Grid on Kubernetes

# selenium-hub-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: selenium-hub
  labels:
    app: selenium-hub
spec:
  replicas: 1
  selector:
    matchLabels:
      app: selenium-hub
  template:
    metadata:
      labels:
        app: selenium-hub
    spec:
      containers:
      - name: selenium-hub
        image: selenium/hub:4.15.0
        ports:
        - containerPort: 4444
        - containerPort: 4442
        - containerPort: 4443
        env:
        - name: SE_SESSION_REQUEST_TIMEOUT
          value: "300"
        - name: SE_NODE_MAX_SESSIONS
          value: "5"
        resources:
          requests:
            memory: "512Mi"
            cpu: "500m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
---
apiVersion: v1
kind: Service
metadata:
  name: selenium-hub
spec:
  selector:
    app: selenium-hub
  ports:
  - name: web
    port: 4444
    targetPort: 4444
  - name: event-bus-publish
    port: 4442
    targetPort: 4442
  - name: event-bus-subscribe
    port: 4443
    targetPort: 4443
  type: ClusterIP

Chrome Node Deployment

# chrome-node-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: chrome-node
  labels:
    app: chrome-node
spec:
  replicas: 5
  selector:
    matchLabels:
      app: chrome-node
  template:
    metadata:
      labels:
        app: chrome-node
    spec:
      containers:
      - name: chrome-node
        image: selenium/node-chrome:4.15.0
        env:
        - name: SE_EVENT_BUS_HOST
          value: "selenium-hub"
        - name: SE_EVENT_BUS_PUBLISH_PORT
          value: "4442"
        - name: SE_EVENT_BUS_SUBSCRIBE_PORT
          value: "4443"
        - name: SE_NODE_MAX_SESSIONS
          value: "3"
        resources:
          requests:
            memory: "1Gi"
            cpu: "1000m"
          limits:
            memory: "2Gi"
            cpu: "2000m"
        volumeMounts:
        - name: dshm
          mountPath: /dev/shm
      volumes:
      - name: dshm
        emptyDir:
          medium: Memory
          sizeLimit: 2Gi

Test Job Configuration

# test-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: selenium-test-job
spec:
  parallelism: 10
  completions: 10
  template:
    spec:
      containers:
      - name: test-runner
        image: your-registry/selenium-tests:latest
        env:
        - name: SELENIUM_GRID_URL
          value: "http://selenium-hub:4444"
        - name: TEST_SUITE
          valueFrom:
            configMapKeyRef:
              name: test-config
              key: suite
        - name: PARALLEL_THREADS
          value: "5"
        resources:
          requests:
            memory: "512Mi"
            cpu: "500m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
      restartPolicy: Never
  backoffLimit: 3

Auto-Scaling Configuration

Horizontal Pod Autoscaler

# hpa.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: chrome-node-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: chrome-node
  minReplicas: 2
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 30
      policies:
      - type: Percent
        value: 100
        periodSeconds: 15
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 50
        periodSeconds: 60

KEDA Scaling (Event-Driven)

# keda-scaledobject.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: selenium-scaler
spec:
  scaleTargetRef:
    name: chrome-node
  minReplicaCount: 2
  maxReplicaCount: 50
  triggers:
  - type: selenium-grid
    metadata:
      url: 'http://selenium-hub:4444/graphql'
      browserName: 'chrome'
      sessionBrowserName: 'chrome'
      targetValue: '1'

Configuration Management

ConfigMap for Test Configuration

# test-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: test-config
data:
  suite: "regression"
  testng.xml: |
    <!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
    <suite name="K8s Suite" parallel="methods" thread-count="10">
      <test name="Regression">
        <packages>
          <package name="com.example.tests.*"/>
        </packages>
      </test>
    </suite>
  application.properties: |
    selenium.grid.url=http://selenium-hub:4444
    app.base.url=https://app.example.com
    test.timeout=30
    screenshot.enabled=true

Secrets for Credentials

# test-secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: test-credentials
type: Opaque
stringData:
  username: "test.user@example.com"
  password: "SecureP@ssw0rd"
  api-key: "your-api-key"

Using Secrets in Tests

public class KubernetesTestBase {
    
    @BeforeClass
    public void setup() {
        // Read from environment variables injected from secrets
        String username = System.getenv("TEST_USERNAME");
        String password = System.getenv("TEST_PASSWORD");
        String apiKey = System.getenv("API_KEY");
        
        // Use in tests
    }
}

CI/CD Integration

GitHub Actions Workflow

# .github/workflows/k8s-tests.yml
name: Kubernetes E2E Tests

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    
    - name: Set up kubectl
      uses: azure/setup-kubectl@v3
      with:
        version: 'latest'
    
    - name: Configure kubectl
      run: |
        echo "${{ secrets.KUBECONFIG }}" | base64 -d > kubeconfig
        export KUBECONFIG=./kubeconfig
    
    - name: Deploy Selenium Grid
      run: |
        kubectl apply -f k8s/selenium-hub-deployment.yaml
        kubectl apply -f k8s/chrome-node-deployment.yaml
        kubectl wait --for=condition=ready pod -l app=selenium-hub --timeout=300s
    
    - name: Build and push test image
      run: |
        docker build -t ${{ secrets.DOCKER_REGISTRY }}/selenium-tests:${{ github.sha }} .
        docker push ${{ secrets.DOCKER_REGISTRY }}/selenium-tests:${{ github.sha }}
    
    - name: Run tests
      run: |
        kubectl create configmap test-config --from-file=testng.xml
        kubectl create -f k8s/test-job.yaml
        kubectl wait --for=condition=complete job/selenium-test-job --timeout=1800s
    
    - name: Get test results
      if: always()
      run: |
        kubectl logs job/selenium-test-job > test-output.log
        kubectl cp $(kubectl get pod -l job-name=selenium-test-job -o jsonpath='{.items[0].metadata.name}'):/app/target/surefire-reports ./reports
    
    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: test-reports
        path: reports/
    
    - name: Cleanup
      if: always()
      run: |
        kubectl delete job selenium-test-job
        kubectl delete configmap test-config
        kubectl delete -f k8s/

Monitoring & Logging

Prometheus Metrics

# prometheus-servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: selenium-grid
spec:
  selector:
    matchLabels:
      app: selenium-hub
  endpoints:
  - port: web
    path: /metrics
    interval: 30s

Grafana Dashboard

{
  "dashboard": {
    "title": "Selenium Grid Metrics",
    "panels": [
      {
        "title": "Active Sessions",
        "targets": [
          {
            "expr": "selenium_grid_hub_sessions_active"
          }
        ]
      },
      {
        "title": "Queue Size",
        "targets": [
          {
            "expr": "selenium_grid_hub_queue_size"
          }
        ]
      },
      {
        "title": "Node Availability",
        "targets": [
          {
            "expr": "selenium_grid_node_total_slots"
          }
        ]
      }
    ]
  }
}

Best Practices

  1. Resource Limits: Always set memory and CPU limits
  2. Health Checks: Implement liveness and readiness probes
  3. Scaling: Use HPA or KEDA for auto-scaling
  4. Storage: Use PersistentVolumes for test artifacts
  5. Networking: Use Services for stable endpoints
  6. Secrets: Never hardcode credentials
  7. Monitoring: Implement comprehensive observability

Key Takeaways

  • Docker containers provide consistent test environments
  • Kubernetes orchestrates distributed test execution
  • Auto-scaling handles variable test loads
  • ConfigMaps and Secrets manage configuration
  • CI/CD pipelines automate deployment
  • Monitoring ensures system health

Practice Exercise

Deploy a complete Selenium Grid on Kubernetes:

  1. Create Docker images for your tests
  2. Deploy Selenium Hub and nodes to Kubernetes
  3. Configure HPA for auto-scaling
  4. Implement test job with parallelism
  5. Set up ConfigMaps and Secrets
  6. Create CI/CD pipeline
  7. Add Prometheus/Grafana monitoring

Next: Learn cloud provider integrations (AWS, Azure, GCP) →