# Copyright (c) 2026 Microsoft Corporation. All rights reserved.
# SPDX-License-Identifier: MIT
name: Eval Validation

on:
  workflow_call:
    inputs:
      change-mode:
        description: "Use an exact immutable range or run full unprivileged validation"
        required: false
        type: string
        default: full
      base-sha:
        description: "Immutable base commit used in range mode"
        required: false
        type: string
        default: ""
      head-sha:
        description: "Immutable head commit used in range mode"
        required: false
        type: string
        default: ""
      soft-fail:
        description: "Whether to continue on eval validation failures"
        required: false
        type: boolean
        default: false
      changed-files-only:
        description: "Only run the eval suite when eval-relevant paths change"
        required: false
        type: boolean
        default: true
      ordinary-shard-count:
        description: "Ordinary agent shard count (1 or 4)"
        required: false
        type: number
        default: 4
      instruction-shard-count:
        description: "Instruction shard count (1 or 2)"
        required: false
        type: number
        default: 2
      skill-shard-count:
        description: "Skill shard count (1 or 2)"
        required: false
        type: number
        default: 2
      baseline-max-parallel:
        description: "Maximum simultaneous isolated baseline model jobs (1 or 2)"
        required: false
        type: number
        default: 2
      compare-shard-count:
        description: "Concurrent vally compare shards per baseline model job (1, 5, or 7)"
        required: false
        type: number
        default: 7
      advisory-equivalence:
        description: "Run the non-gating advisory baseline-equivalence model lanes"
        required: false
        type: boolean
        default: true
    secrets:
      copilot-github-token:
        description: "GitHub credential exported as COPILOT_GITHUB_TOKEN for the Copilot CLI used by vally eval"
        required: false

# Minimal permissions
permissions:
  contents: read

concurrency:
  group: eval-validation-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event.pull_request.number || github.ref }}
  cancel-in-progress: false

jobs:
  eval-validation:
    name: Eval Validation
    runs-on: ubuntu-latest
    # Lint-only job; observed ~2 min. Without this it inherits the 6h default.
    timeout-minutes: 20
    permissions:
      contents: read
    outputs:
      eval-relevant: ${{ steps.detect.outputs.eval-relevant }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0
          ref: ${{ github.sha }}

      - name: Verify resolved head commit
        if: inputs.change-mode == 'range'
        shell: pwsh
        env:
          EXPECTED_HEAD_SHA: ${{ inputs.head-sha }}
        run: |
          $actualHeadSha = (git rev-parse HEAD).Trim()
          if ($env:EXPECTED_HEAD_SHA -cnotmatch '^(?:[0-9a-f]{40}|[0-9a-f]{64})$' -or $actualHeadSha -cne $env:EXPECTED_HEAD_SHA) {
            Write-Output "::error::Checked-out commit $actualHeadSha does not match the resolved head commit."
            exit 1
          }

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Create logs directory
        run: mkdir -p logs

      - name: Generate immutable eval change set
        if: inputs.change-mode == 'range'
        shell: pwsh
        env:
          INPUT_BASE_SHA: ${{ inputs.base-sha }}
          INPUT_HEAD_SHA: ${{ inputs.head-sha }}
        run: |
          if (-not $env:INPUT_BASE_SHA -or -not $env:INPUT_HEAD_SHA) {
            throw 'Range mode requires immutable base and head commit IDs.'
          }
          pwsh -NoProfile -File scripts/evals/Get-EvalChangeSet.ps1 `
            -BaseRef $env:INPUT_BASE_SHA -HeadRef $env:INPUT_HEAD_SHA -OutFile logs/eval-change-set.json
          if ($LASTEXITCODE -ne 0) { throw 'Eval change-set generation failed; selection is unavailable.' }

      - name: Upload immutable eval change set
        if: inputs.change-mode == 'range'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-change-set
          path: logs/eval-change-set.json
          retention-days: 30
          if-no-files-found: error

      - name: Detect eval-relevant changes
        id: detect
        shell: pwsh
        env:
          INPUT_CHANGE_MODE: ${{ inputs.change-mode }}
          INPUT_CHANGED_FILES_ONLY: ${{ inputs.changed-files-only }}
        run: |
          if ($env:INPUT_CHANGE_MODE -eq 'full' -or $env:INPUT_CHANGED_FILES_ONLY -ne 'true') {
            $relevant = $true
            Write-Output 'Full eval validation selected.'
          } elseif ($env:INPUT_CHANGE_MODE -eq 'range') {
            Import-Module ./scripts/evals/Modules/EvalChangeSet.psm1 -Force
            $changeSet = Read-EvalChangeSet -Path logs/eval-change-set.json
            $relevant = Test-EvalChangeSetRelevance -ChangeSet $changeSet -RepoRoot $PWD.Path
          } else {
            throw "Unsupported change mode: $env:INPUT_CHANGE_MODE"
          }

          if ($relevant) {
            'EVAL_RELEVANT=true' >> $env:GITHUB_ENV
            'eval-relevant=true' >> $env:GITHUB_OUTPUT
            Write-Output 'Eval-relevant paths changed; running full eval suite.'
          } else {
            'EVAL_RELEVANT=false' >> $env:GITHUB_ENV
            'eval-relevant=false' >> $env:GITHUB_OUTPUT
            Write-Output 'No eval-relevant paths changed; skipping eval suite.'
          }

      - name: Run eval vally lint
        if: env.EVAL_RELEVANT == 'true'
        run: |
          npm run ci:eval:lint:vally > eval-vally-output.txt 2>&1 || echo "EVAL_FAILED=true" >> "$GITHUB_ENV"
          cat eval-vally-output.txt
        continue-on-error: true

      - name: Run eval schema lint
        if: env.EVAL_RELEVANT == 'true'
        run: |
          npm run ci:eval:lint:schema > eval-schema-output.txt 2>&1 || echo "EVAL_FAILED=true" >> "$GITHUB_ENV"
          cat eval-schema-output.txt
        continue-on-error: true

      - name: Run eval text lint
        if: env.EVAL_RELEVANT == 'true'
        run: |
          npm run ci:eval:lint:text > eval-text-output.txt 2>&1 || echo "EVAL_FAILED=true" >> "$GITHUB_ENV"
          cat eval-text-output.txt
        continue-on-error: true

      - name: Run eval safety lint
        if: env.EVAL_RELEVANT == 'true'
        run: |
          npm run ci:eval:lint:safety > eval-safety-output.txt 2>&1 || echo "EVAL_FAILED=true" >> "$GITHUB_ENV"
          cat eval-safety-output.txt
        continue-on-error: true

      - name: Upload eval validation results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-validation-results
          path: |
            eval-vally-output.txt
            eval-schema-output.txt
            eval-text-output.txt
            eval-safety-output.txt
            logs/vally-test-safety.json
            logs/eval-spec-validation.json
            logs/eval-spec-text.json
            logs/agent-behavior-spec-drift.diff
          retention-days: 30
          if-no-files-found: ignore

      - name: Add job summary
        if: always()
        run: |
          if [ "${{ env.EVAL_RELEVANT }}" != "true" ] && [ "${{ env.EVAL_RELEVANT }}" != "false" ]; then
            {
              echo "## Eval Validation Results"
              echo "Change selection failed or did not complete. Review the failing step."
            } >> "$GITHUB_STEP_SUMMARY"
          elif [ "${{ env.EVAL_RELEVANT }}" == "false" ]; then
            {
              echo "## Eval Validation Results"
              echo "⏭️ **Status**: Skipped"
              echo ""
              echo "No eval-relevant paths changed."
            } >> "$GITHUB_STEP_SUMMARY"
          elif [ "${{ env.EVAL_FAILED }}" == "true" ]; then
            {
              echo "## Eval Validation Results"
              echo "❌ **Status**: Failed"
              echo ""
              echo "Eval validation violations detected. Please review the artifact for details."
            } >> "$GITHUB_STEP_SUMMARY"
          else
            {
              echo "## Eval Validation Results"
              echo "✅ **Status**: Passed"
              echo ""
              echo "No eval validation violations detected."
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Fail job if violations found
        if: env.EVAL_FAILED == 'true' && !inputs.soft-fail
        run: |
          echo "Eval validation failed"
          exit 1

  content-moderation:
    name: Content Moderation
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: eval-validation
    outputs:
      equivalence-required: ${{ steps.artifact-manifest.outputs.equivalence-required }}
    permissions:
      contents: read
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0
          ref: ${{ github.sha }}

      - name: Verify resolved head commit
        if: inputs.change-mode == 'range'
        shell: pwsh
        env:
          EXPECTED_HEAD_SHA: ${{ inputs.head-sha }}
        run: |
          $actualHeadSha = (git rev-parse HEAD).Trim()
          if ($env:EXPECTED_HEAD_SHA -cnotmatch '^(?:[0-9a-f]{40}|[0-9a-f]{64})$' -or $actualHeadSha -cne $env:EXPECTED_HEAD_SHA) {
            Write-Output "::error::Checked-out commit $actualHeadSha does not match the resolved head commit."
            exit 1
          }

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          version: "0.10.9"

      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Sync moderation environment
        run: uv sync --locked
        working-directory: scripts/evals/moderation

      - name: Create logs directory
        run: mkdir -p logs

      - name: Download immutable eval change set
        if: inputs.change-mode == 'range'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: eval-change-set
          path: logs

      - name: Generate changed-artifact manifest
        id: artifact-manifest
        shell: pwsh
        env:
          INPUT_CHANGE_MODE: ${{ inputs.change-mode }}
        run: |
          switch -CaseSensitive ($env:INPUT_CHANGE_MODE) {
            'range' {
              pwsh -NoProfile -File scripts/evals/Get-ChangedAIArtifact.ps1 `
                -ChangeSetPath logs/eval-change-set.json -OutFile logs/changed-ai-artifacts.json
            }
            'full' {
              pwsh -NoProfile -File scripts/evals/Get-ChangedAIArtifact.ps1 `
                -AllTracked -OutFile logs/changed-ai-artifacts.json
            }
            default { throw 'Unsupported change mode.' }
          }
          if ($LASTEXITCODE -ne 0) { throw 'Changed-artifact selection failed.' }
          $manifest = Get-Content -LiteralPath logs/changed-ai-artifacts.json -Raw | ConvertFrom-Json
          $affectedAgents = if ($null -ne $manifest.affectedAgents) { @($manifest.affectedAgents) } else { @() }
          $required = $affectedAgents -contains 'rpi-agent'
          "equivalence-required=$($required.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT

      - name: Run content moderation
        id: moderate
        shell: pwsh
        run: |
          npm run ci:eval:moderate:artifacts
          if ($LASTEXITCODE -ne 0) {
            'MODERATION_FAILED=true' >> $env:GITHUB_ENV
          }
        continue-on-error: true

      - name: Upload moderation results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: content-moderation-results
          path: |
            logs/eval-change-set.json
            logs/changed-ai-artifacts.json
            logs/moderation-artifacts.json
          retention-days: 30
          if-no-files-found: ignore

      - name: Add job summary
        if: always()
        run: |
          if [ "${{ env.MODERATION_FAILED }}" == "true" ]; then
            {
              echo "## Content Moderation Results"
              echo "❌ **Status**: Flagged"
              echo ""
              echo "Content moderation flagged one or more files. Review the moderation-artifacts.json artifact."
            } >> "$GITHUB_STEP_SUMMARY"
          else
            {
              echo "## Content Moderation Results"
              echo "✅ **Status**: Clean"
              echo ""
              echo "No content moderation flags detected."
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Fail job if content flagged
        if: env.MODERATION_FAILED == 'true' && !inputs.soft-fail
        run: |
          echo "Content moderation flagged one or more files"
          exit 1

  agent-plan:
    name: Agent Eval Plan
    runs-on: ubuntu-latest
    needs: [eval-validation, content-moderation]
    if: needs.eval-validation.outputs.eval-relevant == 'true' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    outputs:
      execution-matrix: ${{ steps.plan.outputs.execution-matrix }}
      baseline-required: ${{ steps.plan.outputs.baseline-required }}
      execution-enabled: ${{ steps.plan.outputs.execution-enabled }}
    steps:
      - name: Require a verified change range
        shell: bash
        env:
          INPUT_CHANGE_MODE: ${{ inputs.change-mode }}
        run: |
          if [[ "${INPUT_CHANGE_MODE}" != 'range' ]]; then
            echo "::error::Agent eval selection requires a verified change range; the change-range resolver selected full mode. A manual dispatch from the default branch tip, or from a branch with no commits ahead of it, has no range to verify; dispatch from a branch with changes. Otherwise, investigate the change-range job before rerunning."
            exit 1
          fi

      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Download changed-artifact manifest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: content-moderation-results
          path: logs

      - name: Generate changed-spec stimulus manifest
        shell: pwsh
        run: |
          pwsh -NoProfile -File scripts/evals/Get-ChangedSpecStimulus.ps1 `
            -ChangeSetPath logs/eval-change-set.json -OutFile logs/changed-spec-stimuli.json
          if ($LASTEXITCODE -ne 0) { throw 'Changed-spec selection failed.' }

      - name: Build canonical plan and matrix
        id: plan
        shell: pwsh
        env:
          ORDINARY_SHARD_COUNT: ${{ inputs.ordinary-shard-count || 4 }}
          INSTRUCTION_SHARD_COUNT: ${{ inputs.instruction-shard-count || 2 }}
          SKILL_SHARD_COUNT: ${{ inputs.skill-shard-count || 2 }}
          BASELINE_MAX_PARALLEL: ${{ inputs.baseline-max-parallel || 2 }}
          COMPARE_SHARD_COUNT: ${{ inputs.compare-shard-count || 7 }}
          COPILOT_GITHUB_TOKEN: ${{ secrets.copilot-github-token }}
        run: |
          if ($env:ORDINARY_SHARD_COUNT -notin @('1', '4')) {
            throw "ordinary-shard-count must be 1 or 4; received '$env:ORDINARY_SHARD_COUNT'."
          }
          if ($env:INSTRUCTION_SHARD_COUNT -notin @('1', '2')) {
            throw "instruction-shard-count must be 1 or 2; received '$env:INSTRUCTION_SHARD_COUNT'."
          }
          if ($env:SKILL_SHARD_COUNT -notin @('1', '2')) {
            throw "skill-shard-count must be 1 or 2; received '$env:SKILL_SHARD_COUNT'."
          }
          if ($env:BASELINE_MAX_PARALLEL -notin @('1', '2')) {
            throw "baseline-max-parallel must be 1 or 2; received '$env:BASELINE_MAX_PARALLEL'."
          }
          if ($env:COMPARE_SHARD_COUNT -notin @('1', '5', '7')) {
            throw "compare-shard-count must be 1, 5, or 7; received '$env:COMPARE_SHARD_COUNT'."
          }
          pwsh -NoProfile -File scripts/evals/New-AgentEvalPlan.ps1 `
            -ManifestPath logs/changed-ai-artifacts.json `
            -ChangedSpecManifestPath logs/changed-spec-stimuli.json `
            -OutputPath logs/agent-eval-plan.json `
            -OrdinaryShardCount ([int]$env:ORDINARY_SHARD_COUNT) `
            -InstructionShardCount ([int]$env:INSTRUCTION_SHARD_COUNT) `
            -SkillShardCount ([int]$env:SKILL_SHARD_COUNT)
          if ($LASTEXITCODE -ne 0) { throw 'Canonical agent eval planning failed.' }
          $plan = Get-Content -LiteralPath logs/agent-eval-plan.json -Raw | ConvertFrom-Json -Depth 50
          $include = [System.Collections.Generic.List[object]]::new()
          $include.Add([ordered]@{ kind = 'prompt'; producer = 'prompt'; shard = '' })
          # Matrix order determines job creation order, so emit the heaviest shards before the concurrency cap fills.
          $orderedShards = @($plan.ordinaryShards | Sort-Object @{ Expression = { [int]$_.expectedTrialWeight }; Descending = $true }, @{ Expression = { [string]$_.id }; Ascending = $true })
          foreach ($shard in $orderedShards) {
            $include.Add([ordered]@{ kind = [string]$shard.kind; producer = [string]$shard.id; shard = [string]$shard.id })
          }
          $matrix = [ordered]@{ include = @($include) } | ConvertTo-Json -Depth 5 -Compress
          "execution-matrix=$matrix" >> $env:GITHUB_OUTPUT
          "baseline-required=$(([bool]$plan.baseline.required).ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT
          "execution-enabled=$((-not [string]::IsNullOrWhiteSpace($env:COPILOT_GITHUB_TOKEN)).ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT

      - name: Upload canonical agent eval plan
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agent-eval-plan
          path: |
            logs/changed-ai-artifacts.json
            logs/changed-spec-stimuli.json
            logs/agent-eval-plan.json
          retention-days: 30
          if-no-files-found: error

  eval-execute:
    name: Eval Execute (${{ matrix.producer }})
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs: [eval-validation, content-moderation, agent-plan]
    # Repository secrets are not exposed to fork PRs; clean-skip execution there.
    # Only run when eval-relevant paths changed (see eval-validation detect step).
    if: >-
      needs.eval-validation.outputs.eval-relevant == 'true' &&
      needs.agent-plan.outputs.execution-enabled == 'true' &&
      ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    strategy:
      fail-fast: false
      max-parallel: 6
      matrix: ${{ fromJSON(needs.agent-plan.outputs.execution-matrix) }}
    env:
      COPILOT_GITHUB_TOKEN: ${{ secrets.copilot-github-token }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          version: "0.10.9"

      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Sync moderation environment
        run: uv sync --locked
        working-directory: scripts/evals/moderation

      - name: Create logs directory
        run: mkdir -p logs

      - name: Download canonical agent eval plan
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: agent-eval-plan
          path: logs

      - name: Check Copilot token availability
        id: token-guard
        shell: pwsh
        run: |
          "COPILOT_HOME=$env:RUNNER_TEMP/copilot-home" >> $env:GITHUB_ENV
          if ([string]::IsNullOrWhiteSpace($env:COPILOT_GITHUB_TOKEN)) {
            Write-Output 'COPILOT_GITHUB_TOKEN is not configured; skipping eval execution.'
            'EVAL_EXECUTE=false' >> $env:GITHUB_ENV
          } else {
            'EVAL_EXECUTE=true' >> $env:GITHUB_ENV
          }

      - name: Probe Copilot token
        if: env.EVAL_EXECUTE == 'true'
        shell: pwsh
        run: pwsh -NoProfile -File scripts/evals/Test-CopilotToken.ps1 -SmokeTest

      - name: Check stimulus presence
        if: env.EVAL_EXECUTE == 'true'
        run: npm run ci:eval:presence

      - name: Execute evals
        id: execute
        if: env.EVAL_EXECUTE == 'true'
        shell: pwsh
        env:
          MATRIX_KIND: ${{ matrix.kind }}
          MATRIX_SHARD: ${{ matrix.shard }}
        run: |
          $arguments = @(
            '-Kind', $env:MATRIX_KIND,
            '-ChangedSpecManifestPath', 'logs/changed-spec-stimuli.json',
            '-Model', 'gpt-6-luna',
            '-SkipInputModeration'
          )
          if (-not [string]::IsNullOrWhiteSpace($env:MATRIX_SHARD)) {
            $arguments += @('-PlanPath', 'logs/agent-eval-plan.json', '-ShardId', $env:MATRIX_SHARD)
          }
          npm run ci:eval:execute -- @arguments
          if ($LASTEXITCODE -ne 0) {
            'EVAL_EXECUTE_FAILED=true' >> $env:GITHUB_ENV
          }
        continue-on-error: true

      - name: Upload eval execution results
        if: always() && env.EVAL_EXECUTE == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-execution-results-${{ matrix.producer }}
          # Aggregate results only. Raw agent transcripts and judge logs are withheld
          # because artifacts on a public repository are readable by anyone and a
          # transcript can contain anything the agent printed, including job environment.
          path: |
            logs/changed-ai-artifacts.json
            logs/changed-spec-stimuli.json
            logs/stimulus-presence.json
            logs/eval-summary.json
            logs/eval-results-*.json
          retention-days: 30
          if-no-files-found: error

      - name: Fail job if eval execution failed
        if: env.EVAL_EXECUTE == 'true' && env.EVAL_EXECUTE_FAILED == 'true' && !inputs.soft-fail
        run: |
          echo "Eval execution failed"
          exit 1

  equivalence-execute:
    name: Baseline Equivalence (${{ matrix.model }})
    runs-on: ubuntu-latest
    timeout-minutes: 120
    needs: [eval-validation, content-moderation, agent-plan]
    if: >-
      needs.eval-validation.outputs.eval-relevant == 'true' &&
      needs.agent-plan.outputs.execution-enabled == 'true' &&
      needs.agent-plan.outputs.baseline-required == 'true' &&
      ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    strategy:
      fail-fast: false
      max-parallel: ${{ inputs.baseline-max-parallel || 2 }}
      matrix:
        include:
          - model: gpt-6-luna
            key: gpt
          - model: claude-sonnet-5
            key: claude
    env:
      COPILOT_GITHUB_TOKEN: ${{ secrets.copilot-github-token }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Create logs directory
        run: mkdir -p logs

      - name: Download canonical agent eval plan
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: agent-eval-plan
          path: plan-input

      - name: Check Copilot token availability
        shell: pwsh
        run: |
          "COPILOT_HOME=$env:RUNNER_TEMP/copilot-home" >> $env:GITHUB_ENV
          if ([string]::IsNullOrWhiteSpace($env:COPILOT_GITHUB_TOKEN)) {
            throw 'COPILOT_GITHUB_TOKEN is required for baseline equivalence.'
          }

      - name: Probe Copilot token
        shell: pwsh
        run: pwsh -NoProfile -File scripts/evals/Test-CopilotToken.ps1 -SmokeTest

      - name: Execute isolated model
        id: execute
        shell: pwsh
        env:
          SELECTED_MODEL: ${{ matrix.model }}
          COMPARE_SHARD_COUNT: ${{ inputs.compare-shard-count || 7 }}
          WORKFLOW_RUN_ID: ${{ github.run_id }}
          WORKFLOW_RUN_ATTEMPT: ${{ github.run_attempt }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: |
          $summaryPath = "logs/baseline-equivalence-$env:SELECTED_MODEL.json"
          Import-Module ./scripts/evals/Modules/VallyRunner.psm1 -Force
          $plan = Get-Content -LiteralPath plan-input/agent-eval-plan.json -Raw | ConvertFrom-Json -Depth 50
          if (-not (Test-AgentEvalPlanDigest -Plan $plan) -or -not [bool]$plan.baseline.required -or $env:SELECTED_MODEL -notin @($plan.baseline.models)) {
            throw 'Selected baseline model is not authorized by the canonical agent eval plan.'
          }
          npm run ci:eval:equivalence -- -Agent rpi-agent -Tier calibration `
            -CalibrationModel $env:SELECTED_MODEL `
            -CompareShardCount ([int]$env:COMPARE_SHARD_COUNT) `
            -RepoRoot $PWD -OutputPath $summaryPath
          $producerExitCode = $LASTEXITCODE
          $summary = $null
          if (Test-Path -LiteralPath $summaryPath -PathType Leaf) {
            try { $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json }
            catch { Write-Output 'Equivalence summary is unreadable; fan-in will fail closed.' }
          }
          $driverRunId = if ($null -ne $summary -and $summary.PSObject.Properties['runId']) { [string]$summary.runId } else { $null }
          [ordered]@{
            envelopeSchemaVersion = '1.0.0'
            workflowRunId = $env:WORKFLOW_RUN_ID
            workflowRunAttempt = [int]$env:WORKFLOW_RUN_ATTEMPT
            headSha = $env:HEAD_SHA
            agent = 'rpi-agent'
            tier = 'calibration'
            selectedModel = $env:SELECTED_MODEL
            planDigest = $plan.planDigest
            driverRunId = $driverRunId
            producerExitCode = $producerExitCode
            summary = $summary
          } | ConvertTo-Json -Depth 100 | Set-Content -LiteralPath "logs/model-evidence-$env:SELECTED_MODEL.json" -Encoding utf8NoBOM
          if ($producerExitCode -ne 0) { 'EQUIVALENCE_FAILED=true' >> $env:GITHUB_ENV }

      - name: Upload isolated model evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-equivalence-${{ matrix.key }}
          path: |
            logs/model-evidence-${{ matrix.model }}.json
            logs/baseline-equivalence-${{ matrix.model }}.json
          retention-days: 30
          if-no-files-found: error

      - name: Fail job if model evidence failed
        if: env.EQUIVALENCE_FAILED == 'true' && !inputs.soft-fail
        run: |
          echo "Baseline equivalence failed for ${{ matrix.model }}"
          exit 1

  # Advisory only: this job reports how additional models behave on the baseline
  # equivalence suite. It never gates the PR, no fan-in lists it in `needs`, and its
  # artifact name matches no fan-in download pattern.
  equivalence-advisory:
    name: Advisory Equivalence (${{ matrix.model }})
    runs-on: ubuntu-latest
    timeout-minutes: 120
    continue-on-error: true
    needs: [eval-validation, content-moderation, agent-plan]
    if: >-
      inputs.advisory-equivalence != false &&
      needs.eval-validation.outputs.eval-relevant == 'true' &&
      needs.agent-plan.outputs.execution-enabled == 'true' &&
      needs.agent-plan.outputs.baseline-required == 'true' &&
      ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - model: mai-code-1.1-flash
            key: mai
    env:
      COPILOT_GITHUB_TOKEN: ${{ secrets.copilot-github-token }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Setup PowerShell modules
        uses: ./.github/actions/setup-ps-modules

      - name: Create logs directory
        run: mkdir -p logs

      - name: Check Copilot token availability
        shell: pwsh
        run: |
          "COPILOT_HOME=$env:RUNNER_TEMP/copilot-home" >> $env:GITHUB_ENV
          if ([string]::IsNullOrWhiteSpace($env:COPILOT_GITHUB_TOKEN)) {
            throw 'COPILOT_GITHUB_TOKEN is required for advisory equivalence.'
          }

      - name: Probe Copilot token
        shell: pwsh
        run: pwsh -NoProfile -File scripts/evals/Test-CopilotToken.ps1 -SmokeTest

      - name: Execute advisory model
        shell: pwsh
        env:
          ADVISORY_MODEL: ${{ matrix.model }}
          COMPARE_SHARD_COUNT: ${{ inputs.compare-shard-count || 7 }}
        run: |
          npm run ci:eval:equivalence -- -Agent rpi-agent -Tier devloop `
            -Model $env:ADVISORY_MODEL `
            -CompareShardCount ([int]$env:COMPARE_SHARD_COUNT) `
            -NoBaselineCache `
            -RepoRoot $PWD -OutputPath "logs/advisory-equivalence-$env:ADVISORY_MODEL.json"

      - name: Summarize advisory result
        if: always()
        shell: pwsh
        env:
          ADVISORY_MODEL: ${{ matrix.model }}
        run: |
          $summaryPath = "logs/advisory-equivalence-$env:ADVISORY_MODEL.json"
          $lines = [System.Collections.Generic.List[string]]::new()
          $lines.Add("## Advisory Equivalence: $env:ADVISORY_MODEL")
          $lines.Add('')
          $lines.Add('> Advisory only. This result does not gate the pull request.')
          $lines.Add('')
          $summary = $null
          if (Test-Path -LiteralPath $summaryPath -PathType Leaf) {
            try { $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json }
            catch { $summary = $null }
          }
          if ($null -eq $summary) {
            $lines.Add('No readable advisory summary was produced.')
          }
          else {
            # Only fixed field values and fixed diagnostic categories are rendered;
            # diagnostic text can carry model output and is never published.
            $categories = @($summary.executionDiagnostics | ForEach-Object { @($_.errors) } | ForEach-Object { [string]$_.category } | Sort-Object -Unique)
            $lines.Add('| Field | Value |')
            $lines.Add('|-------|-------|')
            foreach ($field in @('verdict', 'equivalenceGate', 'runs', 'ties', 'baselineWins', 'treatmentWins', 'equivalentTies', 'equivalentTrials', 'invariantFailures', 'invocationFailures', 'judgeErrors', 'dataQualityViolations', 'runHealthFailures')) {
              $lines.Add("| $field | $($summary.$field) |")
            }
            $lines.Add("| executionErrorCategories | $(if ($categories.Count -gt 0) { $categories -join ', ' } else { 'none' }) |")
          }
          $lines -join "`n" >> $env:GITHUB_STEP_SUMMARY

      - name: Upload advisory summary
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: advisory-equivalence-${{ matrix.key }}
          path: logs/advisory-equivalence-${{ matrix.model }}.json
          retention-days: 30
          if-no-files-found: warn

  equivalence-fan-in:
    name: Baseline Equivalence Fan-In
    runs-on: ubuntu-latest
    needs: [eval-validation, content-moderation, agent-plan, equivalence-execute]
    if: >-
      always() && !cancelled() &&
      needs.eval-validation.outputs.eval-relevant == 'true' &&
      needs.agent-plan.outputs.execution-enabled == 'true' &&
      ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Download isolated model evidence
        if: needs.agent-plan.outputs.baseline-required == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: model-results
          pattern: eval-equivalence-*
        continue-on-error: true

      - name: Download canonical agent eval plan
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: agent-eval-plan
          path: plan-input

      - name: Merge expected model evidence
        shell: pwsh
        env:
          EQUIVALENCE_REQUIRED: ${{ needs.agent-plan.outputs.baseline-required }}
          WORKFLOW_RUN_ID: ${{ github.run_id }}
          WORKFLOW_RUN_ATTEMPT: ${{ github.run_attempt }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          SOFT_FAIL: ${{ inputs.soft-fail }}
        run: |
          New-Item -ItemType Directory -Path logs -Force | Out-Null
          $arguments = @(
            '-NoProfile', '-File', 'scripts/evals/Merge-BaselineEquivalence.ps1',
            '-PlanPath', 'plan-input/agent-eval-plan.json',
            '-OutputPath', 'logs/baseline-equivalence-rpi-agent.json',
            '-EvalSummaryPath', 'logs/eval-summary.json',
            '-StatusPath', 'logs/merge-status.json'
          )
          if ($env:EQUIVALENCE_REQUIRED -notin @('true', 'false')) {
            throw "Invalid equivalence-required state '$env:EQUIVALENCE_REQUIRED'."
          }
          if ($env:EQUIVALENCE_REQUIRED -eq 'true') {
            $arguments += @(
              '-EnvelopeDirectory', 'model-results',
              '-ExpectedWorkflowRunId', $env:WORKFLOW_RUN_ID,
              '-ExpectedWorkflowRunAttempt', $env:WORKFLOW_RUN_ATTEMPT,
              '-ExpectedHeadSha', $env:HEAD_SHA
            )
          } else {
            $arguments += '-NotRequired'
          }
          & pwsh @arguments
          $mergeExitCode = $LASTEXITCODE
          if ($mergeExitCode -ne 0) {
            $status = if (Test-Path -LiteralPath logs/merge-status.json) { Get-Content -LiteralPath logs/merge-status.json -Raw | ConvertFrom-Json } else { $null }
            $softEvidenceFailure = $null -ne $status -and $status.category -eq 'evidence' -and $env:SOFT_FAIL -eq 'true'
            if (-not $softEvidenceFailure) { throw 'Baseline equivalence fan-in failed closed.' }
          }
          $fragment = Get-Content -LiteralPath logs/eval-summary.json -Raw | ConvertFrom-Json -Depth 100
          $plan = Get-Content -LiteralPath plan-input/agent-eval-plan.json -Raw | ConvertFrom-Json -Depth 50
          $fragment | Add-Member -NotePropertyName producer -NotePropertyValue 'equivalence' -Force
          $fragment | Add-Member -NotePropertyName planDigest -NotePropertyValue $plan.planDigest -Force
          $fragment | ConvertTo-Json -Depth 100 | Set-Content -LiteralPath logs/eval-summary.json -Encoding utf8NoBOM

      - name: Upload combined equivalence evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-equivalence-combined
          path: |
            logs/baseline-equivalence-rpi-agent.json
            logs/eval-summary.json
            logs/merge-status.json
          retention-days: 30
          if-no-files-found: error

  eval-fan-in:
    name: Authoritative Eval Fan-In
    runs-on: ubuntu-latest
    needs: [eval-validation, agent-plan, eval-execute, equivalence-fan-in]
    if: >-
      always() && !cancelled() &&
      needs.eval-validation.outputs.eval-relevant == 'true' &&
      needs.agent-plan.outputs.execution-enabled == 'true' &&
      ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      github.event_name == 'workflow_dispatch')
    permissions:
      contents: read
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Download canonical agent eval plan
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: agent-eval-plan
          path: plan-input

      - name: Download eval producer summaries
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: producer-results
          pattern: eval-execution-results-*
        continue-on-error: true

      - name: Download combined equivalence summary
        if: needs.agent-plan.outputs.baseline-required == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: producer-results
          pattern: eval-equivalence-combined
        continue-on-error: true

      - name: Merge authoritative eval evidence
        shell: pwsh
        env:
          SOFT_FAIL: ${{ inputs.soft-fail }}
        run: |
          New-Item -ItemType Directory -Path logs -Force | Out-Null
          pwsh -NoProfile -File scripts/evals/Merge-EvalExecution.ps1 `
            -PlanPath plan-input/agent-eval-plan.json `
            -SummaryDirectory producer-results `
            -OutputPath logs/eval-summary.json `
            -StatusPath logs/eval-fan-in-status.json
          $mergeExitCode = $LASTEXITCODE
          if ($mergeExitCode -ne 0) {
            $status = if (Test-Path -LiteralPath logs/eval-fan-in-status.json) { Get-Content -LiteralPath logs/eval-fan-in-status.json -Raw | ConvertFrom-Json } else { $null }
            $softEvidenceFailure = $null -ne $status -and $status.category -eq 'evidence' -and $env:SOFT_FAIL -eq 'true'
            if (-not $softEvidenceFailure) { throw 'Authoritative eval fan-in failed closed.' }
          }

      - name: Upload authoritative eval result
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-authoritative
          path: |
            logs/eval-summary.json
            logs/eval-fan-in-status.json
          retention-days: 30
          if-no-files-found: error

  eval-report:
    name: Eval Report
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: [eval-fan-in]
    # Present the canonical authoritative summary even when execution failed.
    if: always() && github.event.pull_request.head.repo.full_name == github.repository && github.event_name == 'pull_request'
    permissions:
      contents: read
      pull-requests: write
    steps:
      - name: Download authoritative eval result
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: shard-results/eval-authoritative
          name: eval-authoritative
        continue-on-error: true

      - name: Post aggregated eval summary comment
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            const marker = '<!-- eval-execute-summary -->';
            const root = 'shard-results';

            // Consume the single summary produced by authoritative fan-in.
            const summaries = [];
            if (fs.existsSync(root)) {
              for (const dir of fs.readdirSync(root)) {
                const summaryPath = path.join(root, dir, 'eval-summary.json');
                if (fs.existsSync(summaryPath)) {
                  try {
                    summaries.push(JSON.parse(fs.readFileSync(summaryPath, 'utf8')));
                  } catch (err) {
                    console.log(`Skipping unparseable summary: ${summaryPath} (${err.message})`);
                  }
                }
              }
            }

            let body;
            if (summaries.length === 0) {
              body = `${marker}\n## Eval Execution\n\n⚠️ No eval summary was produced.`;
            } else {
              const totals = { artifacts: 0, specs: 0, assertionsPassed: 0, assertionsFailed: 0, failedSpecs: 0 };
              const artifacts = [];
              for (const s of summaries) {
                const t = s.totals || {};
                totals.artifacts += Number(t.artifacts || 0);
                totals.specs += Number(t.specs || 0);
                totals.assertionsPassed += Number(t.assertionsPassed || 0);
                totals.assertionsFailed += Number(t.assertionsFailed || 0);
                totals.failedSpecs += Number(t.failedSpecs || 0);
                for (const a of (s.perArtifact || [])) { artifacts.push(a); }
              }

              // Split each artifact's assertion failures into blocking (authoritative)
              // vs advisory (non-gating). The producer emits these fields directly;
              // fall back to the artifact status when an older summary omits them.
              const splitFailures = (a) => {
                const failed = Number(a.assertionsFailed || 0);
                let blocking = Number(a.authoritativeFailed);
                let advisory = Number(a.advisoryFailed);
                let tolerated = Number(a.toleratedFailed);
                if (!Number.isFinite(blocking) || !Number.isFinite(advisory)) {
                  if (a.status === 'advisory-fail') { blocking = 0; advisory = failed; }
                  else if (a.status === 'pass' || a.status === 'skipped') { blocking = 0; advisory = 0; }
                  else { blocking = failed; advisory = 0; }
                }
                if (!Number.isFinite(tolerated)) { tolerated = Math.max(0, failed - blocking - advisory); }
                return { blocking, advisory, tolerated };
              };

              let blockingTotal = 0;
              let advisoryTotal = 0;
              let toleratedTotal = 0;
              for (const a of artifacts) {
                const { blocking, advisory, tolerated } = splitFailures(a);
                blockingTotal += blocking;
                advisoryTotal += advisory;
                toleratedTotal += tolerated;
              }

              // Only `failedSpecs` gates the PR. When nothing blocks merge but advisory
              // assertions failed, say so explicitly so the per-row ⚠️/❌ icons are not
              // misread as merge blockers.
              let header;
              if (totals.failedSpecs > 0) {
                header = `❌ **Status**: Failed — ${totals.failedSpecs} spec(s) block merge`;
              } else if (advisoryTotal > 0) {
                header = `✅ **Status**: Passed — no merge-blocking failures (${advisoryTotal} advisory assertion failure(s) present)`;
              } else {
                header = '✅ **Status**: Passed';
              }

              const lines = [
                marker,
                '## Eval Execution',
                '',
                header,
                '',
                `- Artifacts evaluated: ${totals.artifacts}`,
                `- Specs run: ${totals.specs}`,
                `- Assertions passed: ${totals.assertionsPassed}`,
                `- Assertions failed (blocking): ${blockingTotal}`,
                `- Assertions failed (advisory): ${advisoryTotal}`,
                `- Trial dips tolerated by per-stimulus aggregate: ${toleratedTotal}`,
                `- Failed specs (merge-blocking): ${totals.failedSpecs}`,
                '',
              ];

              // Advisory token means come from Vally's native tokenUsage; a null or
              // absent mean (no measured trials, or an older summary) renders as an em dash.
              const formatTokens = (value) => {
                if (value === null || value === undefined || value === '') { return '—'; }
                const n = Number(value);
                return Number.isFinite(n) ? Math.round(n).toLocaleString('en-US') : '—';
              };

              if (artifacts.length > 0) {
                lines.push('| Artifact | Kind | Status | Specs | Passed | Failed (blocking) | Failed (advisory) | Trial dips (tolerated) | Input tokens / trial | Cache-read tokens / trial |');
                lines.push('|----------|------|--------|-------|--------|-------------------|-------------------|------------------------|----------------------|---------------------------|');
                for (const a of artifacts) {
                  const icon = a.status === 'pass' ? '✅' : a.status === 'skipped' ? '⏭️' : a.status === 'advisory-fail' ? '⚠️' : '❌';
                  const { blocking, advisory, tolerated } = splitFailures(a);
                  lines.push(`| \`${a.artifactId}\` | ${a.kind} | ${icon} ${a.status} | ${a.specCount || 0} | ${a.assertionsPassed} | ${blocking} | ${advisory} | ${tolerated} | ${formatTokens(a.meanInputTokensPerTrial)} | ${formatTokens(a.meanCacheReadTokensPerTrial)} |`);
                }
                lines.push('');
                lines.push('> **Legend** — ✅ clean · ⚠️ advisory failures only (non-blocking) · ⏭️ skipped · ❌ merge-blocking failure');
                lines.push('>');
                lines.push('> Only **Failed specs (merge-blocking)** gates this PR. Advisory assertion failures are signal-quality checks captured during iteration; review them, but they do not block merge and may be acceptable.');
                lines.push('>');
                lines.push('> Token columns are advisory means per trial reported by the model provider. They vary between trials and never gate this PR.');
              } else {
                lines.push('No changed AI artifacts required evaluation.');
              }

              body = lines.join('\n');
            }

            const { owner, repo } = context.repo;
            const issue_number = context.payload.pull_request.number;
            const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number });
            const existing = comments.find((c) => c.body && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
            } else {
              await github.rest.issues.createComment({ owner, repo, issue_number, body });
            }