📈 Benchmark #353
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 📈 Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| benchmark_types: | |
| description: 'Benchmark types to run (comma-separated)' | |
| required: false | |
| default: 'lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage' | |
| type: string | |
| frameworks: | |
| description: 'Frameworks to test (comma-separated, blank for all)' | |
| required: false | |
| default: '' | |
| type: string | |
| executions: | |
| description: 'Number of executions per benchmark (for averaging)' | |
| required: false | |
| default: '1' | |
| type: string | |
| timeout_minutes: | |
| description: 'Job timeout in minutes' | |
| required: false | |
| default: '60' | |
| type: string | |
| commit_to_main: | |
| description: 'Commit results to main branch (results branch will always be updated)' | |
| required: false | |
| default: false | |
| type: boolean | |
| schedule: | |
| - cron: '30 4 * * *' # Daily at 04:30 UTC | |
| env: | |
| NODE_VERSION: '22' | |
| PYTHON_VERSION: '3.11' | |
| permissions: | |
| contents: read | |
| jobs: | |
| benchmark: | |
| name: Run Framework Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: ${{ fromJson(github.event.inputs.timeout_minutes || '60') }} | |
| steps: | |
| - name: 📥 Checkout Repository | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 | |
| with: | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - name: 🔧 Setup Node.js | |
| uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'npm' | |
| - name: 🐍 Setup Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install System Dependencies | |
| run: | | |
| # Update package lists | |
| sudo apt-get update | |
| # Install Chrome dependencies for Lighthouse (with Ubuntu 24.04 compatibility) | |
| sudo apt-get install -y \ | |
| wget \ | |
| gnupg \ | |
| ca-certificates \ | |
| fonts-liberation \ | |
| libatk-bridge2.0-0 \ | |
| libdrm2 \ | |
| libgtk-3-0 \ | |
| libnspr4 \ | |
| libnss3 \ | |
| libxcomposite1 \ | |
| libxdamage1 \ | |
| libxrandr2 \ | |
| xdg-utils \ | |
| libgbm1 \ | |
| libxss1 | |
| # Install audio library (try both old and new package names) | |
| sudo apt-get install -y libasound2t64 || sudo apt-get install -y libasound2 | |
| - name: 🌐 Install Google Chrome | |
| uses: browser-actions/setup-chrome@c785b87e244131f27c9f19c1a33e2ead956ab7ce # v1 | |
| with: | |
| chrome-version: stable | |
| id: setup-chrome | |
| - name: 📋 Verify Chrome Installation | |
| env: | |
| CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} | |
| run: | | |
| echo "Chrome path: $CHROME_PATH" | |
| "$CHROME_PATH" --version | |
| which google-chrome || which chromium-browser || echo "Chrome executable not found in PATH" | |
| - name: 🔧 Setup Project | |
| run: | | |
| echo "Installing root dependencies..." | |
| npm ci | |
| echo "Installing Python dependencies..." | |
| pip install -r scripts/requirements.txt | |
| echo "Setting up project with npm run setup..." | |
| npm run setup | |
| - name: 🏗️ Build All Framework Apps | |
| run: npm run build | |
| - name: 🚀 Start Development Server | |
| run: | | |
| echo "Starting development server..." | |
| npm start & | |
| # Wait for server to be ready with timeout | |
| timeout=60 | |
| elapsed=0 | |
| while [ $elapsed -lt $timeout ]; do | |
| if curl -sf http://localhost:3000/health > /dev/null 2>&1; then | |
| echo "✅ Server is ready!" | |
| break | |
| fi | |
| echo "⏳ Waiting for server... (${elapsed}s)" | |
| sleep 2 | |
| elapsed=$((elapsed + 2)) | |
| done | |
| if [ $elapsed -ge $timeout ]; then | |
| echo "❌ Server failed to start within ${timeout}s" | |
| echo "Checking server logs..." | |
| jobs | |
| exit 1 | |
| fi | |
| timeout-minutes: 2 | |
| - name: 🔍 Verify Project Setup | |
| continue-on-error: true | |
| run: | | |
| echo "Verifying project setup (skipping tests since server is running)..." | |
| cd scripts && python verify/main.py --skip-test | |
| echo "Testing server endpoints..." | |
| # Test health endpoint | |
| curl -f http://localhost:3000/health || { | |
| echo "❌ Health endpoint failed" | |
| exit 1 | |
| } | |
| # Test a few framework endpoints | |
| for framework in react vue svelte; do | |
| echo "Testing $framework endpoint..." | |
| curl -f "http://localhost:3000/$framework/?mock=true" > /dev/null || { | |
| echo "⚠️ $framework endpoint failed, but continuing..." | |
| } | |
| done | |
| echo "✅ Project verification complete" | |
| - name: 🔧 Prepare Benchmark Environment | |
| env: | |
| COMMIT_TO_MAIN_INPUT: ${{ github.event.inputs.commit_to_main || 'false' }} | |
| BENCHMARK_TYPES_INPUT: ${{ github.event.inputs.benchmark_types || 'all' }} | |
| FRAMEWORKS_INPUT: ${{ github.event.inputs.frameworks || 'all' }} | |
| EXECUTIONS_INPUT: ${{ github.event.inputs.executions || '1' }} | |
| run: | | |
| echo "PYTHONPATH=$PWD:$PWD/scripts" >> "$GITHUB_ENV" | |
| # Set commit behavior based on event type or manual input | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "COMMIT_TO_MAIN=false" >> "$GITHUB_ENV" | |
| echo "⏰ Scheduled run - will skip main branch commit" | |
| echo "SCHEDULED_RUN=true" >> "$GITHUB_ENV" | |
| else | |
| echo "COMMIT_TO_MAIN=$COMMIT_TO_MAIN_INPUT" >> "$GITHUB_ENV" | |
| echo "👤 Manual run - commit to main: $COMMIT_TO_MAIN_INPUT" | |
| echo "SCHEDULED_RUN=false" >> "$GITHUB_ENV" | |
| fi | |
| echo "📊 Benchmark Configuration:" | |
| if [ "$SCHEDULED_RUN" = "true" ]; then | |
| echo " Types: lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage (scheduled)" | |
| echo " Frameworks: all (scheduled)" | |
| echo " Executions: 5 (scheduled)" | |
| else | |
| echo " Types: $BENCHMARK_TYPES_INPUT" | |
| echo " Frameworks: $FRAMEWORKS_INPUT" | |
| echo " Executions: $EXECUTIONS_INPUT" | |
| fi | |
| echo " Commit to Main: $COMMIT_TO_MAIN" | |
| - name: 🧪 Run Benchmarks | |
| env: | |
| BENCHMARK_TYPES_INPUT: ${{ github.event.inputs.benchmark_types }} | |
| FRAMEWORKS_INPUT: ${{ github.event.inputs.frameworks }} | |
| EXECUTIONS_INPUT: ${{ github.event.inputs.executions }} | |
| run: | | |
| set -e # Exit on error | |
| # Configure parameters based on trigger type | |
| if [ "$SCHEDULED_RUN" = "true" ]; then | |
| # Scheduled run: comprehensive benchmarks | |
| benchmark_types="lighthouse,bundle-size,source-analysis,build-time,dev-server,resource-usage" | |
| frameworks="" # All frameworks | |
| executions="5" | |
| echo "⏰ Scheduled run configuration: all benchmarks, all frameworks, 5 executions" | |
| else | |
| # Manual run: use provided inputs | |
| benchmark_types="$BENCHMARK_TYPES_INPUT" | |
| frameworks="$FRAMEWORKS_INPUT" | |
| executions="$EXECUTIONS_INPUT" | |
| fi | |
| # Build command arguments for the benchmark script | |
| cmd_args=() | |
| cmd_args+=("all") # Run all benchmarks by default | |
| if [ -n "$benchmark_types" ]; then | |
| cmd_args+=("--type" "$benchmark_types") | |
| fi | |
| if [ -n "$frameworks" ]; then | |
| cmd_args+=("--frameworks" "$frameworks") | |
| fi | |
| if [ "$executions" != "1" ] && [ -n "$executions" ]; then | |
| cmd_args+=("--executions" "$executions") | |
| fi | |
| echo "🚀 Running benchmarks: npm run benchmark ${cmd_args[*]}" | |
| # Run benchmarks with retry logic | |
| max_retries=2 | |
| retry=0 | |
| while [ $retry -le $max_retries ]; do | |
| if [ $retry -gt 0 ]; then | |
| echo "⏳ Retry attempt $retry/$max_retries..." | |
| sleep 10 | |
| fi | |
| if npm run benchmark -- "${cmd_args[@]}"; then | |
| echo "✅ Benchmarks completed successfully!" | |
| break | |
| else | |
| exit_code=$? | |
| echo "❌ Benchmarks failed with exit code $exit_code" | |
| if [ $retry -eq $max_retries ]; then | |
| echo "💥 All retry attempts exhausted" | |
| exit $exit_code | |
| fi | |
| retry=$((retry + 1)) | |
| # Clean up any hanging processes before retry | |
| pkill -f "chrome" || true | |
| pkill -f "chromium" || true | |
| sleep 5 | |
| fi | |
| done | |
| timeout-minutes: 45 | |
| - name: 📊 Generate Benchmark Summary | |
| if: always() | |
| run: | | |
| { | |
| echo "## 📊 Benchmark Results Summary" | |
| echo "" | |
| echo "- **Run Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # Check if benchmarks ran successfully | |
| if [ -d "benchmark-results" ] && [ "$(find benchmark-results -name "*.json" -type f | wc -l)" -gt 0 ]; then | |
| result_count=$(find benchmark-results -name "*.json" -type f | wc -l) | |
| latest_dir=$(find benchmark-results -mindepth 1 -maxdepth 1 -type d | sort -r | head -1) | |
| { | |
| echo "- **Status**: ✅ Success" | |
| echo "- **Results Generated**: $result_count files" | |
| echo "- **Artifacts**: Available for download below" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if [ -n "$latest_dir" ]; then | |
| echo "- **Latest Results**: $(basename "$latest_dir")" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| else | |
| { | |
| echo "- **Status**: ❌ Failed" | |
| echo "- **Reason**: Benchmarks did not complete successfully" | |
| echo "- **Check**: Review the workflow logs for error details" | |
| echo "- **Artifacts**: Debug logs may be available below (if any)" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: 📋 Save Workflow Context | |
| run: | | |
| echo "{\"commit_to_main\": \"${COMMIT_TO_MAIN}\"}" > workflow-context.json | |
| cat workflow-context.json | |
| - name: 📤 Upload Benchmark Results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: benchmark-results-${{ github.run_number }} | |
| path: | | |
| benchmark-results/ | |
| workflow-context.json | |
| !benchmark-results/**/.gitkeep | |
| retention-days: 30 | |
| compression-level: 6 | |
| - name: 📤 Upload Detailed Logs | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: failure() | |
| with: | |
| name: benchmark-logs-${{ github.run_number }} | |
| path: | | |
| **/*.log | |
| **/npm-debug.log* | |
| **/yarn-debug.log* | |
| **/yarn-error.log* | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: 🧹 Cleanup | |
| if: always() | |
| run: | | |
| echo "🧹 Cleaning up processes..." | |
| pkill -f "npm.*start" || true | |
| pkill -f "chrome" || true | |
| pkill -f "chromium" || true | |
| echo "✅ Cleanup complete" |