From 86f755c179d54d84025ebb0aadcd234d9d49e299 Mon Sep 17 00:00:00 2001
From: Abdul Fatir <Abdulfatirs@gmail.com>
Date: Mon, 2 Dec 2024 11:47:29 +0100
Subject: [PATCH] Fix auto eval workflow (#224)

*Issue #, if available:*

*Description of changes:* This PR fixes the auto evaluation workflow.
The second workflow step did not work because it did not know the right
PR number to post the comment on. The fix is to include the PR number in
the CSV file name and read it in the second workflow.

PS: This is a really poor user experience because there's no way to test
that this works right without merging!


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Abdul Fatir Ansari <ansarnd@amazon.de>
---
 .github/workflows/eval-model.yml      | 11 +++++++++--
 .github/workflows/eval-pr-comment.yml | 11 +++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/eval-model.yml b/.github/workflows/eval-model.yml
index 0446870..95f7e27 100644
--- a/.github/workflows/eval-model.yml
+++ b/.github/workflows/eval-model.yml
@@ -5,11 +5,18 @@ on:
   # Runs only with read privilages for the GITHUB_TOKEN
   pull_request:
     branches: ["main"] # Run on PRs to main branch
+    types:
+      - opened      # When a PR is created
+      - reopened    # When a closed PR is reopened
+      - synchronize # When new commits are pushed to the PR
+      - labeled     # When a label is added to the PR
 
 jobs:
   evaluate-and-post:
     if: contains(github.event.pull_request.labels.*.name, 'run-eval')  # Only run if 'run-eval' label is added
     runs-on: ubuntu-latest
+    env:
+      RESULTS_CSV: "eval-ci-metrics-${{ github.event.pull_request.number }}.csv"
 
     steps:
       - name: Checkout Repository
@@ -24,12 +31,12 @@ jobs:
         run: pip install ".[evaluation]" -f https://download.pytorch.org/whl/cpu/torch_stable.html
 
       - name: Run Eval Script
-        run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml eval-ci-metrics.csv --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32
+        run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml $RESULTS_CSV --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32
       
       - name: Upload CSV
         uses: actions/upload-artifact@v4
         with:
           name: eval-metrics
-          path: eval-ci-metrics.csv
+          path: ${{ env.RESULTS_CSV }}
           retention-days: 1
           overwrite: true
diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml
index 1f0d18c..56b84ae 100644
--- a/.github/workflows/eval-pr-comment.yml
+++ b/.github/workflows/eval-pr-comment.yml
@@ -29,12 +29,19 @@ jobs:
 
       - name: Display structure of downloaded files
         run: ls -R
+      
+      - name: Extract PR number from file name
+        id: extract-pr
+        run: |
+          FILE_NAME=$(ls eval-metrics-artifact/eval-ci-metrics-*.csv)
+          PR_NUMBER=$(echo $FILE_NAME | sed -E 's/.*eval-ci-metrics-(.*).csv/\1/')
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
 
       - name: Read CSV
         id: csv
         uses: juliangruber/read-file-action@v1
         with:
-          path: eval-metrics-artifact/eval-ci-metrics.csv
+          path: eval-metrics-artifact/eval-ci-metrics-${{ env.PR_NUMBER }}.csv
 
       - name: Create Markdown Table
         uses: petems/csv-to-md-table-action@master
@@ -47,7 +54,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           repository: ${{ github.repository }}
-          issue-number: ${{ github.event.pull_request.number }}
+          issue-number: ${{ env.PR_NUMBER }}
           body: |
             ### Evaluation Metrics
             ${{steps.csv-table-output.outputs.markdown-table}}