scheduled-smoke-test.yml

name: Smoke test

on:
  # regular runs
  schedule:
    - cron: '0 0,6,12,18 * * *'
  # allow button click
  workflow_dispatch:

permissions:
  contents: read

jobs:
  test:
    runs-on: ubuntu-latest
    environment: Scheduled Testing

    steps:
    - uses: actions/checkout@v3

    - name: Store commit
      run: |
        echo "GIT_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV

    - name: Set up Python 3.10
      uses: actions/setup-python@v3
      with:
        python-version: "3.10"

    - name: cache poetry install
      uses: actions/cache@v3
      id: cache-poetry
      with:
        path: ~/.local
        key: poetry-1.7.1-0

    - name: Install and configure Poetry
      uses: snok/install-poetry@v1
      with:
        version: 1.7.1
        virtualenvs-create: true
        virtualenvs-in-project: true

    - name: cache deps
      id: cache-deps
      uses: actions/cache@v3
      with:
        path: .venv
        key: pydeps-${{ hashFiles('**/poetry.lock') }}

    - name: Install dependencies with caching
      run: poetry install --no-interaction --no-root
      if: steps.cache-deps.outputs.cache-hit != 'true'

    - name: Install with plugins
      run: poetry install --no-interaction --sync --extras all_plugins

    - name: Write secrets
      env:
        SECRETS_CONFIG: |
          [anthropic]
          api_key = "${{ secrets.ANTHROPIC_API_KEY }}"

          [together]
          api_key = "${{ secrets.TOGETHER_API_KEY }}"

          [openai]
          api_key = "${{ secrets.OPENAI_API_KEY }}"

          [hugging_face]
          token = "${{ secrets.HUGGING_FACE_TOKEN }}"
          
          [google_ai]
          api_key = "${{ secrets.GOOGLE_AI_API_KEY }}"

          [demo]
          api_key="12345"
      run: |
        mkdir -p config
        echo "$SECRETS_CONFIG" > config/secrets.toml

    - name: Test object creation with plugins
      run: |
        source .venv/bin/activate
        pytest --expensive-tests

    - name: Test v0.5 run
      run: |
        source .venv/bin/activate
        modelbench benchmark -m 1 -v 0.5

    - name: Test v1.0 run
      run: |
        source .venv/bin/activate
        modelbench benchmark -m 1 -v 1.0

    - name: Ensure the artifact published on Pypi still works as expected
      run: |
        rm -rf .venv
        mkdir -p ../installation/config
        cat ./tests/modelgauge_tests/data/install_pyproject.toml > ../installation/pyproject.toml
        cd ../installation
        touch ./config/secrets.toml
        poetry lock
        poetry install --no-root
        poetry run modelgauge list-tests

    - name: Discord notification
      if: failure()
      env:
        DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
      uses: Ilshidur/action-discord@0.3.2
      with:
        args: 'The smoke test for {{ EVENT_PAYLOAD.repository.full_name }} has failed. See <{{ EVENT_PAYLOAD.repository.html_url }}/actions>'