Skip to content

Commit b7705d4

Browse files
authored
[wip]: Add retries for setup-matrix (#590)
* use network retries * test different approach to handle canary * force test to fail * more cleanup * force canary to fail * t * refactor ci file * revert test changes * remove unused import * revert quotes change * cleanup * remove redundant comment
1 parent d0137cf commit b7705d4

File tree

3 files changed

+178
-154
lines changed

3 files changed

+178
-154
lines changed

.github/workflows/integration-tests.yml

Lines changed: 4 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ jobs:
1616
timeout-minutes: 30
1717
outputs:
1818
pluginDirs: ${{ steps.set-plugin-dirs.outputs.pluginDirs }}
19-
canaryVersion: ${{ steps.npm-canary-version.outputs.version }}
2019
canaryDockerTag: ${{ steps.docker-canary-tag.outputs.result }}
2120
latestVersion: ${{ steps.npm-latest-version.outputs.version }}
2221
steps:
@@ -27,18 +26,13 @@ jobs:
2726
- name: Setup plugin dir variable
2827
id: set-plugin-dirs
2928
run: echo "pluginDirs=$(find ./examples -type d -name "src" -not -path "*/node_modules*" -maxdepth 3 -exec test -e "{}/plugin.json" \; -print | sed "s/\/src$//" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
30-
- name: Setup NPM canary version variable
31-
id: npm-canary-version
32-
run: echo "version=$(npm view @grafana/ui dist-tags.canary)" >> $GITHUB_OUTPUT
3329
- name: Setup docker canary tag variable
3430
id: docker-canary-tag
3531
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
36-
env:
37-
INPUT_NPM-TAG: ${{ steps.npm-canary-version.outputs.version }}
3832
with:
3933
result-encoding: string
4034
script: |
41-
const script = require('./.github/workflows/scripts/npm-to-docker-image.js');
35+
const script = require('./.github/workflows/scripts/get-dev-image-tag.js');
4236
return await script({ core });
4337
- name: Setup NPM latest version variable
4438
id: npm-latest-version
@@ -120,20 +114,17 @@ jobs:
120114
workdir: ${{ matrix.pluginDir }}
121115
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
122116

123-
## CANARY_VERSION has to use NPM as we only publish the packages on changes
124117
- name: Set environment vars for testing
125118
run: |
126119
echo "PLUGIN_ID=$(cat src/plugin.json | jq -r '.id')" >> $GITHUB_ENV
127120
echo "EXPECTED_GRAFANA_VERSION=$(npx semver@latest $(cat src/plugin.json | jq -r '.dependencies.grafanaDependency') -c)" >> $GITHUB_ENV
128-
echo "CANARY_VERSION=$CANARY_VERSION" >> $GITHUB_ENV
129121
echo "CANARY_DOCKER_TAG=$CANARY_DOCKER_TAG" >> $GITHUB_ENV
130122
echo "LATEST_STABLE_VERSION=$LATEST_STABLE_VERSION" >> $GITHUB_ENV
131123
if [ -f "${PWD}/.env" ]; then
132124
echo "ENV_FILE_OPTION=--env-file ${PWD}/.env" >> $GITHUB_ENV
133125
fi
134126
working-directory: ${{ matrix.pluginDir }}
135127
env:
136-
CANARY_VERSION: ${{ needs.setup-matrix.outputs.canaryVersion }}
137128
CANARY_DOCKER_TAG: ${{ needs.setup-matrix.outputs.canaryDockerTag }}
138129
LATEST_STABLE_VERSION: ${{ needs.setup-matrix.outputs.latestVersion }}
139130

@@ -257,35 +248,9 @@ jobs:
257248
docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
258249
working-directory: ${{ matrix.pluginDir }}
259250

260-
## Canary Version Tests
261-
## Runs the plugin tests against the latest build of Grafana main branch.
262-
263-
- name: Canary - Upgrade @grafana packages using canary_version
264-
run: |
265-
npm install --legacy-peer-deps $(echo $(cat package.json | jq -r --arg version $CANARY_VERSION '["@grafana/eslint-config", "@grafana/tsconfig", "@grafana/scenes", "@grafana/experimental", "@grafana/plugin-e2e", "@grafana/plugin-meta-extractor", "@grafana/eslint-plugin-plugins"] as $blacklist | [(.devDependencies,.dependencies) | keys] | flatten | unique | map(select ( test("^@grafana") ) ) as $deps | $deps - $blacklist | map("\(.)@\($version)") | join(" ")') )
266-
npm install --force --save-optional @swc/core @swc/core-linux-arm-gnueabihf @swc/core-linux-arm64-gnu @swc/core-linux-arm64-musl @swc/core-linux-x64-gnu @swc/core-linux-x64-musl
267-
working-directory: ${{ matrix.pluginDir }}
268-
269-
- name: Canary - Run frontend tests
270-
run: |
271-
npm run test:ci
272-
working-directory: ${{ matrix.pluginDir }}
273-
274-
- name: Canary - Build plugin with grafana dependencies
275-
run: |
276-
npm run build
277-
working-directory: ${{ matrix.pluginDir }}
278-
279-
- name: Canary - Build plugin backend
280-
uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b # v3.1.0
281-
with:
282-
version: latest
283-
args: -v build:linux
284-
workdir: ${{ matrix.pluginDir }}
285-
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
286-
287-
# Canary versions live at grafana/grafana-dev
288-
- name: Canary - Start Grafana dev image
251+
# Canary Version Tests
252+
# Runs the plugin tests against the canary stable version of Grafana.
253+
- name: Canary - Start Grafana
289254
if: steps.has-integration-tests.outputs.DIR == 'true'
290255
run: |
291256
docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana-dev:$CANARY_DOCKER_TAG
Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
const https = require('https');
2+
3+
const DOCKERHUB_API_URL = 'https://registry.hub.docker.com/v2/repositories/grafana/grafana-dev/tags?page_size=25';
4+
const GRAFANA_DEV_TAG_REGEX = /^(\d+\.\d+\.\d+)-(\d+)$/;
5+
const HTTP_TIMEOUT_MS = 10000;
6+
const RETRYABLE_ERROR_CODES = ['ECONNRESET', 'ENOTFOUND', 'ECONNREFUSED', 'ETIMEDOUT'];
7+
8+
/**
9+
* Main entry point
10+
*/
11+
module.exports = async ({ core }) => {
12+
try {
13+
console.log('Getting latest Grafana dev tag from DockerHub...');
14+
15+
const latestTag = await getLatestGrafanaDevTag();
16+
17+
if (!latestTag) {
18+
core.setFailed('Could not find any Grafana dev tags on DockerHub');
19+
return;
20+
}
21+
22+
core.info(`Found grafana/grafana-dev:${latestTag}`);
23+
return latestTag;
24+
} catch (error) {
25+
core.setFailed(error.message);
26+
}
27+
};
28+
29+
/**
30+
* Fetches and returns the latest Grafana dev tag from DockerHub
31+
* @returns {Promise<string|null>} Latest tag name or null if not found
32+
*/
33+
async function getLatestGrafanaDevTag() {
34+
try {
35+
console.log('Fetching latest 25 tags from DockerHub...');
36+
const response = await httpGet(DOCKERHUB_API_URL);
37+
38+
if (!response?.results?.length) {
39+
console.log('No tags found');
40+
return null;
41+
}
42+
43+
console.log(`Found ${response.results.length} tags`);
44+
45+
const validTags = response.results
46+
.map((item) => item.name)
47+
.map(parseGrafanaDevTag)
48+
.filter(Boolean)
49+
.sort((a, b) => b.buildNumber - a.buildNumber);
50+
51+
if (validTags.length === 0) {
52+
console.log('No valid Grafana dev tags found');
53+
return null;
54+
}
55+
56+
const latestTag = validTags[0];
57+
console.log(`Latest tag: ${latestTag.tag} (build ${latestTag.buildNumber}, from ${validTags.length} valid tags)`);
58+
return latestTag.tag;
59+
} catch (error) {
60+
console.log(`Error getting latest tag: ${error.message}`);
61+
return null;
62+
}
63+
}
64+
65+
/**
66+
* Parses a Grafana dev tag string and extracts version and build information
67+
* @param {string} tagName - Tag name to parse (e.g., "12.3.0-17948569556")
68+
* @returns {Object|null} Parsed tag info or null if invalid
69+
*/
70+
function parseGrafanaDevTag(tagName) {
71+
const match = tagName.match(GRAFANA_DEV_TAG_REGEX);
72+
if (!match) {
73+
return null;
74+
}
75+
76+
return {
77+
tag: tagName,
78+
version: match[1],
79+
buildNumber: parseInt(match[2], 10),
80+
};
81+
}
82+
83+
/**
84+
* Makes an HTTP GET request with retry logic
85+
* @param {string} url - URL to fetch
86+
* @param {number} maxRetries - Maximum number of retry attempts
87+
* @param {number} retryDelay - Base delay between retries in milliseconds
88+
* @returns {Promise<Object>} Parsed JSON response
89+
*/
90+
function httpGet(url, maxRetries = 10, retryDelay = 2000) {
91+
return new Promise((resolve, reject) => {
92+
let attempts = 0;
93+
let timeoutId = null;
94+
95+
const clearRetryTimeout = () => {
96+
if (timeoutId) {
97+
clearTimeout(timeoutId);
98+
timeoutId = null;
99+
}
100+
};
101+
102+
const scheduleRetry = (error) => {
103+
if (attempts < maxRetries && isRetryableError(error)) {
104+
const delay = retryDelay * attempts;
105+
console.warn(`Retrying ${url} (attempt ${attempts}/${maxRetries}) in ${delay}ms: ${error.message}`);
106+
timeoutId = setTimeout(makeRequest, delay);
107+
} else {
108+
reject(error);
109+
}
110+
};
111+
112+
const makeRequest = () => {
113+
attempts++;
114+
115+
const req = https.get(url, { timeout: HTTP_TIMEOUT_MS }, (res) => {
116+
const chunks = [];
117+
118+
res.on('data', (chunk) => chunks.push(chunk));
119+
120+
res.on('end', () => {
121+
clearRetryTimeout();
122+
const responseBody = Buffer.concat(chunks).toString();
123+
124+
if (res.statusCode >= 200 && res.statusCode < 300) {
125+
try {
126+
resolve(JSON.parse(responseBody));
127+
} catch (parseError) {
128+
const error = new Error(`Failed to parse JSON from ${url}: ${parseError.message}`);
129+
error.responseBody = responseBody.substring(0, 500);
130+
scheduleRetry(error);
131+
}
132+
} else if (res.statusCode >= 500) {
133+
const error = new Error(`Server error ${res.statusCode} from ${url}`);
134+
error.statusCode = res.statusCode;
135+
scheduleRetry(error);
136+
} else {
137+
const error = new Error(`HTTP ${res.statusCode} error from ${url}`);
138+
error.statusCode = res.statusCode;
139+
error.responseBody = responseBody.substring(0, 500);
140+
reject(error);
141+
}
142+
});
143+
144+
res.on('error', (err) => {
145+
clearRetryTimeout();
146+
scheduleRetry(err);
147+
});
148+
});
149+
150+
req.on('timeout', () => {
151+
req.destroy();
152+
const error = new Error(`Request timeout for ${url}`);
153+
error.code = 'ETIMEDOUT';
154+
scheduleRetry(error);
155+
});
156+
157+
req.on('error', (err) => {
158+
clearRetryTimeout();
159+
scheduleRetry(err);
160+
});
161+
};
162+
163+
makeRequest();
164+
});
165+
}
166+
167+
/**
168+
* Determines if an error is retryable
169+
* @param {Error} error - Error to check
170+
* @returns {boolean} True if error is retryable
171+
*/
172+
function isRetryableError(error) {
173+
return RETRYABLE_ERROR_CODES.includes(error.code) || error.statusCode >= 500;
174+
}

.github/workflows/scripts/npm-to-docker-image.js

Lines changed: 0 additions & 115 deletions
This file was deleted.

0 commit comments

Comments
 (0)