Skip to content

Commit cdd3f82

Browse files
per1234per1234
committed
Set up basic integration tests
Although the action's Python code itself is currently well tested via unit tests, this does not provide any checks for how the action interacts with the GitHub Actions framework and GitHub API. Even though it won't provide full verification of the report, simply running the action provides a "smoke test" as well as facilitating manual verification. On every pull request event, the action will run and source the sketches reports test data from a local path. When the job is triggered by an event from a fork, it will fail due to the access token not having the write permissions required to comment on the PR thread. The workflow is triggered to still pass under these conditions. Checking the log to see whether the action failed in the expected manner may still provide a useful validation of the PR. When the PR is from a branch of the repository, the deltas report will be commented on the PR thread. The workflow is also configured to run on push, schedule, workflow_dispatch, and repository_dispatch events. In this case it sources the sketches report from a workflow artifact. A dummy PR will be left permanently open in the repository with the sole purpose of serving as a target for these tests.
1 parent 6162636 commit cdd3f82

File tree

5 files changed

+790
-0
lines changed

5 files changed

+790
-0
lines changed

‎.github/workflows/test-integration.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Run integration tests
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- '.github/workflows/test-integration.yml'
7+
- '.github/workflows/testdata/**'
8+
- 'action.yml'
9+
- 'Dockerfile'
10+
- 'reportsizedeltas/**'
11+
push:
12+
paths:
13+
- '.github/workflows/test-integration.yml'
14+
- '.github/workflows/testdata/**'
15+
- 'action.yml'
16+
- 'Dockerfile'
17+
- 'reportsizedeltas/**'
18+
schedule:
19+
# Run daily at 8 AM UTC to catch breakage caused by changes to external resources.
20+
- cron: "0 8 * * *"
21+
workflow_dispatch:
22+
repository_dispatch:
23+
24+
jobs:
25+
local-source:
26+
if: github.event_name == 'pull_request'
27+
runs-on: ubuntu-latest
28+
29+
steps:
30+
- name: Checkout repository
31+
uses: actions/checkout@v2
32+
33+
- name: Run action
34+
# Use arduino/report-size-deltas action from local path
35+
uses: ./
36+
# The action will always fail on PRs submitted from forks due to not having write permissions.
37+
# Some verification can still be achieved by checking the log to see whether it failed in the expected manner:
38+
# WARNING:__main__:Temporarily unable to open URL (HTTP Error 403: Forbidden), retrying
39+
# ...
40+
# TimeoutError: Maximum number of URL load retries exceeded
41+
continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
42+
with:
43+
sketches-reports-source: .github/workflows/testdata/sketches-reports
44+
45+
artifact-source:
46+
if: github.event_name != 'pull_request'
47+
runs-on: ubuntu-latest
48+
steps:
49+
- name: Checkout repository
50+
uses: actions/checkout@v2
51+
52+
- name: Run action
53+
# Use arduino/report-size-deltas action from local path
54+
uses: ./

‎.github/workflows/testdata/sketches-reports/arduino-avr-leonardo.json

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
{
2+
"commit_hash": "651f05f4d4aca30ac359e972c01568f873112d43",
3+
"commit_url": "https://github.com/per1234/generate-size-deltas-report/commit/651f05f4d4aca30ac359e972c01568f873112d43",
4+
"boards": [
5+
{
6+
"board": "arduino:avr:leonardo",
7+
"sketches": [
8+
{
9+
"name": "examples/MIDIUSB_clock",
10+
"compilation_success": true,
11+
"sizes": [
12+
{
13+
"name": "flash",
14+
"maximum": 28672,
15+
"current": {
16+
"absolute": 4792,
17+
"relative": 16.71
18+
},
19+
"previous": {
20+
"absolute": 4792,
21+
"relative": 16.71
22+
},
23+
"delta": {
24+
"absolute": 0,
25+
"relative": 0.0
26+
}
27+
},
28+
{
29+
"name": "RAM for global variables",
30+
"maximum": 2560,
31+
"current": {
32+
"absolute": 443,
33+
"relative": 17.3
34+
},
35+
"previous": {
36+
"absolute": 443,
37+
"relative": 17.3
38+
},
39+
"delta": {
40+
"absolute": 0,
41+
"relative": 0.0
42+
}
43+
}
44+
],
45+
"warnings": {
46+
"current": {
47+
"absolute": 2
48+
},
49+
"previous": {
50+
"absolute": 2
51+
},
52+
"delta": {
53+
"absolute": 0
54+
}
55+
}
56+
},
57+
{
58+
"name": "examples/MIDIUSB_loop",
59+
"compilation_success": true,
60+
"sizes": [
61+
{
62+
"name": "flash",
63+
"maximum": 28672,
64+
"current": {
65+
"absolute": 4870,
66+
"relative": 16.99
67+
},
68+
"previous": {
69+
"absolute": 4870,
70+
"relative": 16.99
71+
},
72+
"delta": {
73+
"absolute": 0,
74+
"relative": 0.0
75+
}
76+
},
77+
{
78+
"name": "RAM for global variables",
79+
"maximum": 2560,
80+
"current": {
81+
"absolute": 441,
82+
"relative": 17.23
83+
},
84+
"previous": {
85+
"absolute": 441,
86+
"relative": 17.23
87+
},
88+
"delta": {
89+
"absolute": 0,
90+
"relative": 0.0
91+
}
92+
}
93+
],
94+
"warnings": {
95+
"current": {
96+
"absolute": 3
97+
},
98+
"previous": {
99+
"absolute": 3
100+
},
101+
"delta": {
102+
"absolute": 0
103+
}
104+
}
105+
},
106+
{
107+
"name": "examples/MIDIUSB_read",
108+
"compilation_success": true,
109+
"sizes": [
110+
{
111+
"name": "flash",
112+
"maximum": 28672,
113+
"current": {
114+
"absolute": 4908,
115+
"relative": 17.12
116+
},
117+
"previous": {
118+
"absolute": 4908,
119+
"relative": 17.12
120+
},
121+
"delta": {
122+
"absolute": 0,
123+
"relative": 0.0
124+
}
125+
},
126+
{
127+
"name": "RAM for global variables",
128+
"maximum": 2560,
129+
"current": {
130+
"absolute": 457,
131+
"relative": 17.85
132+
},
133+
"previous": {
134+
"absolute": 457,
135+
"relative": 17.85
136+
},
137+
"delta": {
138+
"absolute": 0,
139+
"relative": 0.0
140+
}
141+
}
142+
],
143+
"warnings": {
144+
"current": {
145+
"absolute": 3
146+
},
147+
"previous": {
148+
"absolute": 3
149+
},
150+
"delta": {
151+
"absolute": 0
152+
}
153+
}
154+
},
155+
{
156+
"name": "examples/MIDIUSB_write",
157+
"compilation_success": true,
158+
"sizes": [
159+
{
160+
"name": "flash",
161+
"maximum": 28672,
162+
"current": {
163+
"absolute": 4524,
164+
"relative": 15.78
165+
},
166+
"previous": {
167+
"absolute": 4564,
168+
"relative": 15.92
169+
},
170+
"delta": {
171+
"absolute": -40,
172+
"relative": -0.14
173+
}
174+
},
175+
{
176+
"name": "RAM for global variables",
177+
"maximum": 2560,
178+
"current": {
179+
"absolute": 197,
180+
"relative": 7.7
181+
},
182+
"previous": {
183+
"absolute": 213,
184+
"relative": 8.32
185+
},
186+
"delta": {
187+
"absolute": -16,
188+
"relative": -0.62
189+
}
190+
}
191+
],
192+
"warnings": {
193+
"current": {
194+
"absolute": 3
195+
},
196+
"previous": {
197+
"absolute": 3
198+
},
199+
"delta": {
200+
"absolute": 0
201+
}
202+
}
203+
}
204+
],
205+
"sizes": [
206+
{
207+
"name": "flash",
208+
"maximum": 28672,
209+
"delta": {
210+
"absolute": {
211+
"minimum": -40,
212+
"maximum": 0
213+
},
214+
"relative": {
215+
"minimum": -0.14,
216+
"maximum": 0.0
217+
}
218+
}
219+
},
220+
{
221+
"name": "RAM for global variables",
222+
"maximum": 2560,
223+
"delta": {
224+
"absolute": {
225+
"minimum": -16,
226+
"maximum": 0
227+
},
228+
"relative": {
229+
"minimum": -0.62,
230+
"maximum": 0.0
231+
}
232+
}
233+
}
234+
],
235+
"warnings": {
236+
"delta": {
237+
"absolute": {
238+
"minimum": 0,
239+
"maximum": 0
240+
}
241+
}
242+
}
243+
}
244+
]
245+
}

0 commit comments

Comments
 (0)