-
Notifications
You must be signed in to change notification settings - Fork 2.9k
283 lines (248 loc) · 12.7 KB
/
e2ePerformanceTests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
name: E2E Performance Tests
on:
workflow_call:
inputs:
PR_NUMBER:
description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
type: string
required: true
workflow_dispatch:
inputs:
PR_NUMBER:
description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
type: string
required: true
jobs:
buildBaseline:
runs-on: ubuntu-latest-xl
name: Build apk from latest release as a baseline
outputs:
VERSION: ${{ steps.getMostRecentRelease.outputs.VERSION }}
ARTIFACT_FOUND: ${{ steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND }}
ARTIFACT_WORKFLOW_ID: ${{ steps.checkForExistingArtifact.outputs.ARTIFACT_WORKFLOW_ID }}
steps:
- uses: actions/checkout@v4
with:
# The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API)
token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
- name: Get most recent release version
id: getMostRecentRelease
# run: echo "VERSION=$(gh release list --limit 1 | awk '{ print $1 }')" >> "$GITHUB_OUTPUT"
# TODO: REVERT TO ABOVE LINE; ONLY FOR TESTING
run: echo "VERSION=on_existing_version" >> "$GITHUB_OUTPUT"
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Check if there's an existing artifact for this baseline
id: checkForExistingArtifact
# TODO: CHANGE TO MAIN GITHUB VERSION BEFORE MERGING
uses: ./.github/actions/javascript/getArtifactInfo
with:
GITHUB_TOKEN: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}
- name: Skip build if there's already an existing artifact for the baseline
if: ${{ fromJSON(steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND) }}
run: echo 'APK for baseline ${{ steps.getMostRecentRelease.outputs.VERSION }} already exists, reusing existing build'
- name: Checkout "Baseline" commit (last release)
if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND) }}
run: |
git fetch origin tag ${{ steps.getMostRecentRelease.outputs.VERSION }} --no-tags --depth=1
git switch --detach ${{ steps.getMostRecentRelease.outputs.VERSION }}
- uses: ./.github/actions/composite/buildAndroidE2EAPK
if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND) }}
with:
ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}
PACKAGE_SCRIPT_NAME: android-build-e2e
APP_OUTPUT_PATH: android/app/build/outputs/apk/e2e/release/app-e2e-release.apk
MAPBOX_SDK_DOWNLOAD_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}
EXPENSIFY_PARTNER_NAME: ${{ secrets.EXPENSIFY_PARTNER_NAME }}
EXPENSIFY_PARTNER_PASSWORD: ${{ secrets.EXPENSIFY_PARTNER_PASSWORD }}
EXPENSIFY_PARTNER_USER_ID: ${{ secrets.EXPENSIFY_PARTNER_USER_ID }}
EXPENSIFY_PARTNER_USER_SECRET: ${{ secrets.EXPENSIFY_PARTNER_USER_SECRET }}
EXPENSIFY_PARTNER_PASSWORD_EMAIL: ${{ secrets.EXPENSIFY_PARTNER_PASSWORD_EMAIL }}
PATH_ENV_FILE: tests/e2e/.env.e2e
buildDelta:
runs-on: ubuntu-latest-xl
name: Build apk from delta ref
outputs:
DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }}
steps:
- uses: actions/checkout@v4
- name: Get pull request details
id: getPullRequestDetails
uses: ./.github/actions/javascript/getPullRequestDetails
with:
GITHUB_TOKEN: ${{ github.token }}
PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }}
USER: ${{ github.actor }}
- name: Merged PR - Get merge commit sha for the pull request
if: ${{ fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
id: getMergeCommitShaIfMergedPR
run: |
MERGE_COMMIT_SHA=${{ steps.getPullRequestDetails.outputs.MERGE_COMMIT_SHA }}
git fetch origin "$MERGE_COMMIT_SHA" --no-tags --depth=1
echo "MERGE_COMMIT_SHA=$MERGE_COMMIT_SHA" >> "$GITHUB_OUTPUT"
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Unmerged PR - Fetch head ref of unmerged PR
if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
run: |
git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1
- name: Unmerged PR - Set dummy git credentials before merging
if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
run: |
git config --global user.email "test@test.com"
git config --global user.name "Test"
- name: Unmerged PR - Merge pull request locally and get merge commit sha
if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
id: getMergeCommitShaIfUnmergedPR
run: |
git merge --allow-unrelated-histories --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Determine "delta ref"
id: getDeltaRef
run: echo "DELTA_REF=${{ steps.getMergeCommitShaIfMergedPR.outputs.MERGE_COMMIT_SHA || steps.getMergeCommitShaIfUnmergedPR.outputs.MERGE_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Checkout "delta ref"
run: git checkout ${{ steps.getDeltaRef.outputs.DELTA_REF }}
#TODO: REVERT TO Expensify/App/.github/actions/composite/buildAndroidE2EAPK@main BEFORE MERGING!!!
- uses: ./.github/actions/composite/buildAndroidE2EAPK
with:
ARTIFACT_NAME: delta-apk-${{ steps.getDeltaRef.outputs.DELTA_REF }}
ARTIFACT_RETENTION_DAYS: 3 # We don't need to store the delta apk for long, its only really needed for the next job in this workflow
PACKAGE_SCRIPT_NAME: android-build-e2edelta
APP_OUTPUT_PATH: android/app/build/outputs/apk/e2edelta/release/app-e2edelta-release.apk
MAPBOX_SDK_DOWNLOAD_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}
EXPENSIFY_PARTNER_NAME: ${{ secrets.EXPENSIFY_PARTNER_NAME }}
EXPENSIFY_PARTNER_PASSWORD: ${{ secrets.EXPENSIFY_PARTNER_PASSWORD }}
EXPENSIFY_PARTNER_USER_ID: ${{ secrets.EXPENSIFY_PARTNER_USER_ID }}
EXPENSIFY_PARTNER_USER_SECRET: ${{ secrets.EXPENSIFY_PARTNER_USER_SECRET }}
EXPENSIFY_PARTNER_PASSWORD_EMAIL: ${{ secrets.EXPENSIFY_PARTNER_PASSWORD_EMAIL }}
PATH_ENV_FILE: tests/e2e/.env.e2edelta
runTestsInAWS:
runs-on: ubuntu-latest
needs: [buildBaseline, buildDelta]
name: Run E2E tests in AWS device farm
steps:
- uses: actions/checkout@v4
with:
# The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API)
token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
- name: Setup Node
uses: ./.github/actions/composite/setupNode
- name: Make zip directory for everything to send to AWS Device Farm
run: mkdir zip
- name: Download baseline APK
uses: actions/download-artifact@348754975ef0295bfa2c111cba996120cfdf8a5d
id: downloadBaselineAPK
with:
name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }}
path: zip
# Set github-token only if the baseline was built in this workflow run:
github-token: ${{ needs.buildBaseline.outputs.ARTIFACT_WORKFLOW_ID && github.token }}
run-id: ${{ needs.buildBaseline.outputs.ARTIFACT_WORKFLOW_ID }}
# The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
- name: Rename baseline APK
run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk"
- name: Download delta APK
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
id: downloadDeltaAPK
with:
name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }}
path: zip
- name: Rename delta APK
run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2edelta-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2edeltaRelease.apk"
- name: Copy e2e code into zip folder
run: cp -r tests/e2e zip
- name: Zip everything in the zip directory up
run: zip -qr App.zip ./zip
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Schedule AWS Device Farm test run on main branch
uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
id: schedule-awsdf-main
with:
name: App E2E Performance Regression Tests
project_arn: ${{ secrets.AWS_PROJECT_ARN }}
device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
app_file: zip/app-e2eRelease.apk
app_type: ANDROID_APP
test_type: APPIUM_NODE
test_package_file: App.zip
test_package_type: APPIUM_NODE_TEST_PACKAGE
test_spec_file: tests/e2e/TestSpec.yml
test_spec_type: APPIUM_NODE_TEST_SPEC
remote_src: false
file_artifacts: |
Customer Artifacts.zip
Test spec output.txt
log_artifacts: debug.log
cleanup: true
timeout: 5400
- name: Print logs if run failed
if: failure()
run: |
echo ${{ steps.schedule-awsdf-main.outputs.data }}
unzip "Customer Artifacts.zip" -d mainResults
cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/logcat.txt" || true
cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log || true
cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/Test spec output.txt" || true
- name: Announce failed workflow in Slack
if: failure()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
channel: '#e2e-announce',
attachments: [{
color: 'danger',
text: `💥 ${process.env.AS_REPO} E2E Test run failed failed on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow 💥`,
}]
}
env:
GITHUB_TOKEN: ${{ github.token }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
- name: Unzip AWS Device Farm results
if: always()
run: unzip "Customer Artifacts.zip"
- name: Print AWS Device Farm run results
if: always()
run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
- name: Check if test failed, if so post the results and add the DeployBlocker label
id: checkIfRegressionDetected
run: |
if grep -q '🔴' ./output.md; then
# Create an output to the GH action that the test failed:
echo "performanceRegressionDetected=true" >> "$GITHUB_OUTPUT"
gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
gh pr comment ${{ inputs.PR_NUMBER }} -F ./output.md
gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
else
echo "performanceRegressionDetected=false" >> "$GITHUB_OUTPUT"
echo '✅ no performance regression detected'
fi
env:
GITHUB_TOKEN: ${{ github.token }}
- name: 'Announce regression in Slack'
if: ${{ steps.checkIfRegressionDetected.outputs.performanceRegressionDetected == 'true' }}
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
channel: '#newdot-performance',
attachments: [{
color: 'danger',
text: `🔴 Performance regression detected in PR ${{ inputs.PR_NUMBER }}\nDetected in <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow.`,
}]
}
env:
GITHUB_TOKEN: ${{ github.token }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}