benchmark.yml

name: 📈 Benchmark

on:
  workflow_dispatch:
    inputs:
      benchmark_types:
        description: 'Benchmark types to run (comma-separated)'
        required: false
        default: 'lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage'
        type: string
      frameworks:
        description: 'Frameworks to test (comma-separated, blank for all)'
        required: false
        default: ''
        type: string
      executions:
        description: 'Number of executions per benchmark (for averaging)'
        required: false
        default: '1'
        type: string
      timeout_minutes:
        description: 'Job timeout in minutes'
        required: false
        default: '60'
        type: string
      commit_to_main:
        description: 'Commit results to main branch (results branch will always be updated)'
        required: false
        default: false
        type: boolean
  schedule:
    - cron: '30 4 * * *'  # Daily at 04:30 UTC

env:
  NODE_VERSION: '22'
  PYTHON_VERSION: '3.11'

permissions:
  contents: read

jobs:
  benchmark:
    name: Run Framework Benchmarks
    runs-on: ubuntu-latest
    timeout-minutes: ${{ fromJson(github.event.inputs.timeout_minutes || '60') }}
    
    steps:
      - name: 📥 Checkout Repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          fetch-depth: 1
          persist-credentials: false

      - name: 🔧 Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: 🐍 Setup Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: 📦 Install System Dependencies
        run: |
          # Update package lists
          sudo apt-get update
          
          # Install Chrome dependencies for Lighthouse (with Ubuntu 24.04 compatibility)
          sudo apt-get install -y \
            wget \
            gnupg \
            ca-certificates \
            fonts-liberation \
            libatk-bridge2.0-0 \
            libdrm2 \
            libgtk-3-0 \
            libnspr4 \
            libnss3 \
            libxcomposite1 \
            libxdamage1 \
            libxrandr2 \
            xdg-utils \
            libgbm1 \
            libxss1
          
          # Install audio library (try both old and new package names)
          sudo apt-get install -y libasound2t64 || sudo apt-get install -y libasound2

      - name: 🌐 Install Google Chrome
        uses: browser-actions/setup-chrome@c785b87e244131f27c9f19c1a33e2ead956ab7ce # v1
        with:
          chrome-version: stable
        id: setup-chrome

      - name: 📋 Verify Chrome Installation
        env:
          CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
        run: |
          echo "Chrome path: $CHROME_PATH"
          "$CHROME_PATH" --version
          which google-chrome || which chromium-browser || echo "Chrome executable not found in PATH"

      - name: 🔧 Setup Project
        run: |
          echo "Installing root dependencies..."
          npm ci
          
          echo "Installing Python dependencies..."
          pip install -r scripts/requirements.txt
          
          echo "Setting up project with npm run setup..."
          npm run setup

      - name: 🏗️ Build All Framework Apps
        run: npm run build

      - name: 🚀 Start Development Server
        run: |
          echo "Starting development server..."
          npm start &
          
          # Wait for server to be ready with timeout
          timeout=60
          elapsed=0
          while [ $elapsed -lt $timeout ]; do
            if curl -sf http://localhost:3000/health > /dev/null 2>&1; then
              echo "✅ Server is ready!"
              break
            fi
            echo "⏳ Waiting for server... (${elapsed}s)"
            sleep 2
            elapsed=$((elapsed + 2))
          done
          
          if [ $elapsed -ge $timeout ]; then
            echo "❌ Server failed to start within ${timeout}s"
            echo "Checking server logs..."
            jobs
            exit 1
          fi
        timeout-minutes: 2

      - name: 🔍 Verify Project Setup  
        continue-on-error: true
        run: |
          echo "Verifying project setup (skipping tests since server is running)..."
          cd scripts && python verify/main.py --skip-test
          
          echo "Testing server endpoints..."
          
          # Test health endpoint
          curl -f http://localhost:3000/health || {
            echo "❌ Health endpoint failed"
            exit 1
          }
          
          # Test a few framework endpoints
          for framework in react vue svelte; do
            echo "Testing $framework endpoint..."
            curl -f "http://localhost:3000/$framework/?mock=true" > /dev/null || {
              echo "⚠️ $framework endpoint failed, but continuing..."
            }
          done
          
          echo "✅ Project verification complete"

      - name: 🔧 Prepare Benchmark Environment
        env:
          COMMIT_TO_MAIN_INPUT: ${{ github.event.inputs.commit_to_main || 'false' }}
          BENCHMARK_TYPES_INPUT: ${{ github.event.inputs.benchmark_types || 'all' }}
          FRAMEWORKS_INPUT: ${{ github.event.inputs.frameworks || 'all' }}
          EXECUTIONS_INPUT: ${{ github.event.inputs.executions || '1' }}
        run: |
          echo "PYTHONPATH=$PWD:$PWD/scripts" >> "$GITHUB_ENV"

          # Set commit behavior based on event type or manual input
          if [ "${{ github.event_name }}" = "schedule" ]; then
            echo "COMMIT_TO_MAIN=false" >> "$GITHUB_ENV"
            echo "⏰ Scheduled run - will skip main branch commit"
            echo "SCHEDULED_RUN=true" >> "$GITHUB_ENV"
          else
            echo "COMMIT_TO_MAIN=$COMMIT_TO_MAIN_INPUT" >> "$GITHUB_ENV"
            echo "👤 Manual run - commit to main: $COMMIT_TO_MAIN_INPUT"
            echo "SCHEDULED_RUN=false" >> "$GITHUB_ENV"
          fi

          echo "📊 Benchmark Configuration:"
          if [ "$SCHEDULED_RUN" = "true" ]; then
            echo "  Types: lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage (scheduled)"
            echo "  Frameworks: all (scheduled)"
            echo "  Executions: 5 (scheduled)"
          else
            echo "  Types: $BENCHMARK_TYPES_INPUT"
            echo "  Frameworks: $FRAMEWORKS_INPUT"
            echo "  Executions: $EXECUTIONS_INPUT"
          fi
          echo "  Commit to Main: $COMMIT_TO_MAIN"

      - name: 🧪 Run Benchmarks
        env:
          BENCHMARK_TYPES_INPUT: ${{ github.event.inputs.benchmark_types }}
          FRAMEWORKS_INPUT: ${{ github.event.inputs.frameworks }}
          EXECUTIONS_INPUT: ${{ github.event.inputs.executions }}
        run: |
          set -e  # Exit on error

          # Configure parameters based on trigger type
          if [ "$SCHEDULED_RUN" = "true" ]; then
            # Scheduled run: comprehensive benchmarks
            benchmark_types="lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage"
            frameworks=""  # All frameworks
            executions="5"
            echo "⏰ Scheduled run configuration: all benchmarks, all frameworks, 5 executions"
          else
            # Manual run: use provided inputs
            benchmark_types="$BENCHMARK_TYPES_INPUT"
            frameworks="$FRAMEWORKS_INPUT"
            executions="$EXECUTIONS_INPUT"
          fi
          
          # Build command arguments for the benchmark script
          cmd_args=()
          cmd_args+=("all")  # Run all benchmarks by default
          
          if [ -n "$benchmark_types" ]; then
            cmd_args+=("--type" "$benchmark_types")
          fi
          
          if [ -n "$frameworks" ]; then
            cmd_args+=("--frameworks" "$frameworks")
          fi
          
          if [ "$executions" != "1" ] && [ -n "$executions" ]; then
            cmd_args+=("--executions" "$executions")
          fi

          echo "🚀 Running benchmarks: npm run benchmark ${cmd_args[*]}"
          
          # Run benchmarks with retry logic
          max_retries=2
          retry=0
          
          while [ $retry -le $max_retries ]; do
            if [ $retry -gt 0 ]; then
              echo "⏳ Retry attempt $retry/$max_retries..."
              sleep 10
            fi
            
            if npm run benchmark -- "${cmd_args[@]}"; then
              echo "✅ Benchmarks completed successfully!"
              break
            else
              exit_code=$?
              echo "❌ Benchmarks failed with exit code $exit_code"
              
              if [ $retry -eq $max_retries ]; then
                echo "💥 All retry attempts exhausted"
                exit $exit_code
              fi
              
              retry=$((retry + 1))
              
              # Clean up any hanging processes before retry
              pkill -f "chrome" || true
              pkill -f "chromium" || true
              sleep 5
            fi
          done
          
        timeout-minutes: 45

      - name: 📊 Generate Benchmark Summary
        if: always()
        run: |
          {
            echo "## 📊 Benchmark Results Summary"
            echo ""
            echo "- **Run Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          } >> "$GITHUB_STEP_SUMMARY"
          
          # Check if benchmarks ran successfully
          if [ -d "benchmark-results" ] && [ "$(find benchmark-results -name "*.json" -type f | wc -l)" -gt 0 ]; then
            result_count=$(find benchmark-results -name "*.json" -type f | wc -l)
            latest_dir=$(find benchmark-results -mindepth 1 -maxdepth 1 -type d | sort -r | head -1)
            
            {
              echo "- **Status**: ✅ Success"
              echo "- **Results Generated**: $result_count files"
              echo "- **Artifacts**: Available for download below"
            } >> "$GITHUB_STEP_SUMMARY"
            
            if [ -n "$latest_dir" ]; then
              echo "- **Latest Results**: $(basename "$latest_dir")" >> "$GITHUB_STEP_SUMMARY"
            fi
          else
            {
              echo "- **Status**: ❌ Failed"
              echo "- **Reason**: Benchmarks did not complete successfully"
              echo "- **Check**: Review the workflow logs for error details"
              echo "- **Artifacts**: Debug logs may be available below (if any)"
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: 📋 Save Workflow Context
        run: |
          echo "{\"commit_to_main\": \"${COMMIT_TO_MAIN}\"}" > workflow-context.json
          cat workflow-context.json

      - name: 📤 Upload Benchmark Results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: benchmark-results-${{ github.run_number }}
          path: |
            benchmark-results/
            workflow-context.json
            !benchmark-results/**/.gitkeep
          retention-days: 30
          compression-level: 6

      - name: 📤 Upload Detailed Logs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: benchmark-logs-${{ github.run_number }}
          path: |
            **/*.log
            **/npm-debug.log*
            **/yarn-debug.log*
            **/yarn-error.log*
          retention-days: 7
          if-no-files-found: ignore

      - name: 🧹 Cleanup
        if: always()
        run: |
          echo "🧹 Cleaning up processes..."
          pkill -f "npm.*start" || true
          pkill -f "chrome" || true
          pkill -f "chromium" || true
          echo "✅ Cleanup complete"