Smoke test #202
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test | |
| on: | |
| # regular runs | |
| schedule: | |
| # Every day at 08:00 UTC | |
| - cron: '0 0,8,16 * * *' | |
| # allow button click | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| environment: Scheduled Testing | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Store commit | |
| run: | | |
| echo "GIT_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: cache poetry install | |
| uses: actions/cache@v3 | |
| id: cache-poetry | |
| with: | |
| path: ~/.local | |
| key: poetry-1.7.1-0 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.7.1 | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: cache deps | |
| id: cache-deps | |
| uses: actions/cache@v3 | |
| with: | |
| path: .venv | |
| key: pydeps-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies with caching | |
| run: poetry install --no-interaction --no-root | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| - name: Install with plugins | |
| run: poetry install --no-interaction --sync --extras all_plugins | |
| - name: Write secrets | |
| env: | |
| SECRETS_CONFIG: | | |
| [together] | |
| api_key = "${{ secrets.TOGETHER_API_KEY }}" | |
| [openai] | |
| api_key = "${{ secrets.OPENAI_API_KEY }}" | |
| [hugging_face] | |
| token = "${{ secrets.HUGGING_FACE_TOKEN }}" | |
| [google_ai] | |
| api_key = "${{ secrets.GOOGLE_AI_API_KEY }}" | |
| [demo] | |
| api_key="12345" | |
| run: | | |
| mkdir -p config | |
| echo "$SECRETS_CONFIG" > config/secrets.toml | |
| - name: Test object creation with plugins | |
| run: | | |
| source .venv/bin/activate | |
| pytest --expensive-tests | |
| # TODO Disabled pending Modelbench#509 | |
| # - name: Test standard run | |
| # run: | | |
| # source .venv/bin/activate | |
| # modelbench benchmark --debug -m 1 | |
| # | |
| # - name: Test v1 run | |
| # run: | | |
| # source .venv/bin/activate | |
| # modelbench benchmark -m 1 --benchmark GeneralPurposeAiChatBenchmarkV1 | |
| - name: Ensure the artifact published on Pypi still works as expected | |
| run: | | |
| rm -rf .venv | |
| mkdir -p ../installation/config | |
| cat ./tests/modelgauge_tests/data/install_pyproject.toml > ../installation/pyproject.toml | |
| cd ../installation | |
| touch ./config/secrets.toml | |
| poetry lock | |
| poetry install --no-root | |
| poetry run modelgauge list-tests | |
| - uses: JasonEtco/create-an-issue@v2 | |
| if: failure() | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| with: | |
| filename: .github/failed-scheduled-issue.md | |
| - name: Discord notification | |
| if: failure() | |
| env: | |
| DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} | |
| uses: Ilshidur/action-discord@0.3.2 | |
| with: | |
| args: 'The smoke test for {{ EVENT_PAYLOAD.repository.full_name }} has failed. See {{ EVENT_PAYLOAD.repository.html_url }}/actions' |