# Evaluates Chronos-Bolt (Small) model on selected datasets
name: Evaluate

on:
  # Runs only with read privilages for the GITHUB_TOKEN
  pull_request:
    branches: ["main"] # Run on PRs to main branch
    types:
      - opened      # When a PR is created
      - reopened    # When a closed PR is reopened
      - synchronize # When new commits are pushed to the PR
      - labeled     # When a label is added to the PR


permissions:
  contents: read

jobs:
  evaluate-and-print:
    if: contains(github.event.pull_request.labels.*.name, 'run-eval')  # Only run if 'run-eval' label is added
    runs-on: ubuntu-latest
    env:
      CHRONOS_BOLT_RESULTS_CSV: "eval-ci-metrics-chronos-bolt-${{ github.event.pull_request.number }}.csv"
      CHRONOS_2_RESULTS_CSV: "eval-ci-metrics-chronos-2-${{ github.event.pull_request.number }}.csv"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Dependencies
        run: pip install ".[dev]" -f https://download.pytorch.org/whl/cpu/torch_stable.html

      - name: Run Eval Script for Chronos-2
        run: python scripts/evaluation/evaluate.py chronos-2 ci/evaluate/backtest_config.yaml $CHRONOS_2_RESULTS_CSV --model-id=amazon/chronos-2 --device=cpu --torch-dtype=float32

      - name: Print Chronos-2 CSV
        run: cat $CHRONOS_2_RESULTS_CSV

      - name: Run Eval Script for Chronos-Bolt
        run: python scripts/evaluation/evaluate.py chronos-bolt ci/evaluate/backtest_config.yaml $CHRONOS_BOLT_RESULTS_CSV --model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32

      - name: Print Chronos-Bolt CSV
        run: cat $CHRONOS_BOLT_RESULTS_CSV