mirror of
https://github.com/LukeHagar/vercel.git
synced 2025-12-06 12:57:46 +00:00
Restore datadog checking with logs not exit codes (#10527)
Restores DataDog flakey test detection. Briefly disabled because the old method of using exit codes would bail the entire workflow. Taking the Turbo team's suggestion of looking at `runData.execution` values to avoid an extra loop. Now with unit tests!
This commit is contained in:
Trek Glowacki
2
.changeset/pink-drinks-obey.md
Normal file
2
.changeset/pink-drinks-obey.md
Normal file
@@ -0,0 +1,2 @@
|
||||
---
|
||||
---
|
||||
9
.github/workflows/test.yml
vendored
9
.github/workflows/test.yml
vendored
@@ -91,11 +91,18 @@ jobs:
|
||||
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
|
||||
VERCEL_TEST_REGISTRATION_URL: ${{ secrets.VERCEL_TEST_REGISTRATION_URL }}
|
||||
FORCE_COLOR: '1'
|
||||
- name: 'Determing Turbo HIT or MISS'
|
||||
id: turbo-summary
|
||||
shell: bash
|
||||
run: |
|
||||
TURBO_MISS_COUNT=`node utils/determine-turbo-hit-or-miss.js`
|
||||
echo "MISS COUNT: $TURBO_MISS_COUNT"
|
||||
echo "misses=$TURBO_MISS_COUNT" >> $GITHUB_OUTPUT
|
||||
- name: fetch ssl certificate after tests (linux, os x)
|
||||
if: matrix.runner != 'windows-latest'
|
||||
run: echo | openssl s_client -showcerts -servername 'api.vercel.com' -connect 76.76.21.21:443
|
||||
- name: 'Upload Test Report to Datadog'
|
||||
if: always()
|
||||
if: ${{ steps['turbo-summary'].outputs.misses != '0' }}
|
||||
run: 'npx @datadog/datadog-ci@2.18.1 junit upload --service vercel-cli .junit-reports'
|
||||
env:
|
||||
DATADOG_API_KEY: ${{secrets.DATADOG_API_KEY_CLI}}
|
||||
|
||||
@@ -13,8 +13,8 @@ import { setupTmpDir } from '../../helpers/setup-unit-fixture';
|
||||
|
||||
describe('link', () => {
|
||||
it('should prompt for link', async () => {
|
||||
const cwd = setupTmpDir();
|
||||
const user = useUser();
|
||||
const cwd = setupTmpDir();
|
||||
useTeams('team_dummy');
|
||||
const { project } = useProject({
|
||||
...defaultProject,
|
||||
|
||||
179
test/utils/determine-turbo-hit-or-miss/hit/sample.json
Normal file
179
test/utils/determine-turbo-hit-or-miss/hit/sample.json
Normal file
@@ -0,0 +1,179 @@
|
||||
{
|
||||
"id": "2VLeNEmIfljBVdkudEPZbp5EutG",
|
||||
"version": "1",
|
||||
"turboVersion": "1.10.13",
|
||||
"monorepo": true,
|
||||
"globalCacheInputs": {
|
||||
"rootKey": "You don't understand! I coulda had class. I coulda been a contender. I could've been somebody, instead of a bum, which is what I am.",
|
||||
"files": {
|
||||
"package.json": "5e90f8e7c78d58414bab77fcd16f942f07f426e2",
|
||||
"test/lib/deployment/fetch-retry.js": "f979b34fa9b36be13ecf634e97f085b54f158832",
|
||||
"test/lib/deployment/log.js": "6f750a0cb4cbc59e38d087207d4484a06d83ae51",
|
||||
"test/lib/deployment/now-deploy.js": "a023aab600d7d9ba9da20f96ff060c9ab0bec4c5",
|
||||
"test/lib/deployment/retry-bail-by-default.js": "44830d0f6565655b15fbc441784dc34164749d32",
|
||||
"test/lib/deployment/test-deployment.js": "a18b2cdab36143b27b22313c94fbd74b401bba5a",
|
||||
"test/lib/run-build-lambda.js": "85eba9c6f05ef0584a52d74b4646e1bfbc410484",
|
||||
"turbo-cache-key.json": "523ef4a5572612ec58429b46b5f2b28ac3e24b90",
|
||||
"utils/build.mjs": "074efdecb40a4e4cfdc6acec9439e4632ea1dfa0"
|
||||
},
|
||||
"hashOfExternalDependencies": "a340512aef639cd9",
|
||||
"globalDotEnv": null,
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
}
|
||||
},
|
||||
"packages": ["@vercel/error-utils"],
|
||||
"envMode": "infer",
|
||||
"frameworkInference": true,
|
||||
"execution": {
|
||||
"command": "turbo run test --filter=@vercel/error-utils -- test/index.test.ts",
|
||||
"repoPath": "",
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"cached": 2,
|
||||
"attempted": 2,
|
||||
"startTime": 1694619745822,
|
||||
"endTime": 1694619746625,
|
||||
"exitCode": 0
|
||||
},
|
||||
"tasks": [
|
||||
{
|
||||
"taskId": "@vercel/error-utils#build",
|
||||
"task": "build",
|
||||
"package": "@vercel/error-utils",
|
||||
"hash": "5d8590c998bec75a",
|
||||
"inputs": {
|
||||
"CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
|
||||
"jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
|
||||
"package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
|
||||
"src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
|
||||
"test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
|
||||
"test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
|
||||
"tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
|
||||
},
|
||||
"hashOfExternalDependencies": "1999225753d8a28a",
|
||||
"cache": {
|
||||
"local": false,
|
||||
"remote": true,
|
||||
"status": "HIT",
|
||||
"source": "REMOTE",
|
||||
"timeSaved": 10154
|
||||
},
|
||||
"command": "node ../../utils/build.mjs",
|
||||
"cliArguments": [],
|
||||
"outputs": ["dist/**"],
|
||||
"excludedOutputs": null,
|
||||
"logFile": "packages/error-utils/.turbo/turbo-build.log",
|
||||
"directory": "packages/error-utils",
|
||||
"dependencies": [],
|
||||
"dependents": ["@vercel/error-utils#test"],
|
||||
"resolvedTaskDefinition": {
|
||||
"outputs": ["dist/**"],
|
||||
"cache": true,
|
||||
"dependsOn": ["^build"],
|
||||
"inputs": [],
|
||||
"outputMode": "new-only",
|
||||
"persistent": false,
|
||||
"env": [],
|
||||
"passThroughEnv": null,
|
||||
"dotEnv": null
|
||||
},
|
||||
"expandedOutputs": [
|
||||
"packages/error-utils/.turbo/turbo-build.log",
|
||||
"packages/error-utils/dist",
|
||||
"packages/error-utils/dist/index.d.ts",
|
||||
"packages/error-utils/dist/index.js",
|
||||
"packages/error-utils/dist/index.js.map"
|
||||
],
|
||||
"framework": "",
|
||||
"envMode": "loose",
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
},
|
||||
"dotEnv": null,
|
||||
"execution": {
|
||||
"startTime": 1694619745975,
|
||||
"endTime": 1694619746484,
|
||||
"exitCode": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"taskId": "@vercel/error-utils#test",
|
||||
"task": "test",
|
||||
"package": "@vercel/error-utils",
|
||||
"hash": "5bb22ba520f1cbe8",
|
||||
"inputs": {
|
||||
"CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
|
||||
"jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
|
||||
"package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
|
||||
"src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
|
||||
"test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
|
||||
"test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
|
||||
"tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
|
||||
},
|
||||
"hashOfExternalDependencies": "1999225753d8a28a",
|
||||
"cache": {
|
||||
"local": false,
|
||||
"remote": true,
|
||||
"status": "HIT",
|
||||
"source": "REMOTE",
|
||||
"timeSaved": 4572
|
||||
},
|
||||
"command": "jest --reporters=default --reporters=jest-junit --coverage --env node --verbose",
|
||||
"cliArguments": ["test/index.test.ts"],
|
||||
"outputs": null,
|
||||
"excludedOutputs": null,
|
||||
"logFile": "packages/error-utils/.turbo/turbo-test.log",
|
||||
"directory": "packages/error-utils",
|
||||
"dependencies": ["@vercel/error-utils#build"],
|
||||
"dependents": [],
|
||||
"resolvedTaskDefinition": {
|
||||
"outputs": [],
|
||||
"cache": true,
|
||||
"dependsOn": ["build"],
|
||||
"inputs": [],
|
||||
"outputMode": "new-only",
|
||||
"persistent": false,
|
||||
"env": [],
|
||||
"passThroughEnv": null,
|
||||
"dotEnv": null
|
||||
},
|
||||
"expandedOutputs": ["packages/error-utils/.turbo/turbo-test.log"],
|
||||
"framework": "",
|
||||
"envMode": "loose",
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
},
|
||||
"dotEnv": null,
|
||||
"execution": {
|
||||
"startTime": 1694619746484,
|
||||
"endTime": 1694619746625,
|
||||
"exitCode": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"user": "trek",
|
||||
"scm": {
|
||||
"type": "git",
|
||||
"sha": "0d9d05ba49a4cfda6b60c396b9e84abe34a6c8c1",
|
||||
"branch": "10501/merge"
|
||||
}
|
||||
}
|
||||
13
test/utils/determine-turbo-hit-or-miss/index.test.js
Normal file
13
test/utils/determine-turbo-hit-or-miss/index.test.js
Normal file
@@ -0,0 +1,13 @@
|
||||
const determineTurboHitOrMiss = require('../../../utils/determine-turbo-hit-or-miss');
|
||||
|
||||
describe('determineTurboHitOrMiss', () => {
|
||||
it('detects HITs', async () => {
|
||||
let missCount = await determineTurboHitOrMiss('hit', __dirname);
|
||||
expect(missCount).toEqual(0);
|
||||
});
|
||||
|
||||
it('detects MISSes', async () => {
|
||||
let missCount = await determineTurboHitOrMiss('miss', __dirname);
|
||||
expect(missCount).toEqual(1);
|
||||
});
|
||||
});
|
||||
179
test/utils/determine-turbo-hit-or-miss/miss/sample.json
Normal file
179
test/utils/determine-turbo-hit-or-miss/miss/sample.json
Normal file
@@ -0,0 +1,179 @@
|
||||
{
|
||||
"id": "2VLeNEmIfljBVdkudEPZbp5EutG",
|
||||
"version": "1",
|
||||
"turboVersion": "1.10.13",
|
||||
"monorepo": true,
|
||||
"globalCacheInputs": {
|
||||
"rootKey": "You don't understand! I coulda had class. I coulda been a contender. I could've been somebody, instead of a bum, which is what I am.",
|
||||
"files": {
|
||||
"package.json": "5e90f8e7c78d58414bab77fcd16f942f07f426e2",
|
||||
"test/lib/deployment/fetch-retry.js": "f979b34fa9b36be13ecf634e97f085b54f158832",
|
||||
"test/lib/deployment/log.js": "6f750a0cb4cbc59e38d087207d4484a06d83ae51",
|
||||
"test/lib/deployment/now-deploy.js": "a023aab600d7d9ba9da20f96ff060c9ab0bec4c5",
|
||||
"test/lib/deployment/retry-bail-by-default.js": "44830d0f6565655b15fbc441784dc34164749d32",
|
||||
"test/lib/deployment/test-deployment.js": "a18b2cdab36143b27b22313c94fbd74b401bba5a",
|
||||
"test/lib/run-build-lambda.js": "85eba9c6f05ef0584a52d74b4646e1bfbc410484",
|
||||
"turbo-cache-key.json": "523ef4a5572612ec58429b46b5f2b28ac3e24b90",
|
||||
"utils/build.mjs": "074efdecb40a4e4cfdc6acec9439e4632ea1dfa0"
|
||||
},
|
||||
"hashOfExternalDependencies": "a340512aef639cd9",
|
||||
"globalDotEnv": null,
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
}
|
||||
},
|
||||
"packages": ["@vercel/error-utils"],
|
||||
"envMode": "infer",
|
||||
"frameworkInference": true,
|
||||
"execution": {
|
||||
"command": "turbo run test --filter=@vercel/error-utils -- test/index.test.ts",
|
||||
"repoPath": "",
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"cached": 1,
|
||||
"attempted": 2,
|
||||
"startTime": 1694619745822,
|
||||
"endTime": 1694619746625,
|
||||
"exitCode": 0
|
||||
},
|
||||
"tasks": [
|
||||
{
|
||||
"taskId": "@vercel/error-utils#build",
|
||||
"task": "build",
|
||||
"package": "@vercel/error-utils",
|
||||
"hash": "5d8590c998bec75a",
|
||||
"inputs": {
|
||||
"CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
|
||||
"jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
|
||||
"package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
|
||||
"src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
|
||||
"test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
|
||||
"test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
|
||||
"tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
|
||||
},
|
||||
"hashOfExternalDependencies": "1999225753d8a28a",
|
||||
"cache": {
|
||||
"local": false,
|
||||
"remote": true,
|
||||
"status": "MISS",
|
||||
"source": "REMOTE",
|
||||
"timeSaved": 10154
|
||||
},
|
||||
"command": "node ../../utils/build.mjs",
|
||||
"cliArguments": [],
|
||||
"outputs": ["dist/**"],
|
||||
"excludedOutputs": null,
|
||||
"logFile": "packages/error-utils/.turbo/turbo-build.log",
|
||||
"directory": "packages/error-utils",
|
||||
"dependencies": [],
|
||||
"dependents": ["@vercel/error-utils#test"],
|
||||
"resolvedTaskDefinition": {
|
||||
"outputs": ["dist/**"],
|
||||
"cache": true,
|
||||
"dependsOn": ["^build"],
|
||||
"inputs": [],
|
||||
"outputMode": "new-only",
|
||||
"persistent": false,
|
||||
"env": [],
|
||||
"passThroughEnv": null,
|
||||
"dotEnv": null
|
||||
},
|
||||
"expandedOutputs": [
|
||||
"packages/error-utils/.turbo/turbo-build.log",
|
||||
"packages/error-utils/dist",
|
||||
"packages/error-utils/dist/index.d.ts",
|
||||
"packages/error-utils/dist/index.js",
|
||||
"packages/error-utils/dist/index.js.map"
|
||||
],
|
||||
"framework": "",
|
||||
"envMode": "loose",
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
},
|
||||
"dotEnv": null,
|
||||
"execution": {
|
||||
"startTime": 1694619745975,
|
||||
"endTime": 1694619746484,
|
||||
"exitCode": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"taskId": "@vercel/error-utils#test",
|
||||
"task": "test",
|
||||
"package": "@vercel/error-utils",
|
||||
"hash": "5bb22ba520f1cbe8",
|
||||
"inputs": {
|
||||
"CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
|
||||
"jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
|
||||
"package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
|
||||
"src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
|
||||
"test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
|
||||
"test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
|
||||
"tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
|
||||
},
|
||||
"hashOfExternalDependencies": "1999225753d8a28a",
|
||||
"cache": {
|
||||
"local": false,
|
||||
"remote": true,
|
||||
"status": "HIT",
|
||||
"source": "REMOTE",
|
||||
"timeSaved": 4572
|
||||
},
|
||||
"command": "jest --reporters=default --reporters=jest-junit --coverage --env node --verbose",
|
||||
"cliArguments": ["test/index.test.ts"],
|
||||
"outputs": null,
|
||||
"excludedOutputs": null,
|
||||
"logFile": "packages/error-utils/.turbo/turbo-test.log",
|
||||
"directory": "packages/error-utils",
|
||||
"dependencies": ["@vercel/error-utils#build"],
|
||||
"dependents": [],
|
||||
"resolvedTaskDefinition": {
|
||||
"outputs": [],
|
||||
"cache": true,
|
||||
"dependsOn": ["build"],
|
||||
"inputs": [],
|
||||
"outputMode": "new-only",
|
||||
"persistent": false,
|
||||
"env": [],
|
||||
"passThroughEnv": null,
|
||||
"dotEnv": null
|
||||
},
|
||||
"expandedOutputs": ["packages/error-utils/.turbo/turbo-test.log"],
|
||||
"framework": "",
|
||||
"envMode": "loose",
|
||||
"environmentVariables": {
|
||||
"specified": {
|
||||
"env": [],
|
||||
"passThroughEnv": null
|
||||
},
|
||||
"configured": [],
|
||||
"inferred": [],
|
||||
"passthrough": null
|
||||
},
|
||||
"dotEnv": null,
|
||||
"execution": {
|
||||
"startTime": 1694619746484,
|
||||
"endTime": 1694619746625,
|
||||
"exitCode": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"user": "trek",
|
||||
"scm": {
|
||||
"type": "git",
|
||||
"sha": "0d9d05ba49a4cfda6b60c396b9e84abe34a6c8c1",
|
||||
"branch": "10501/merge"
|
||||
}
|
||||
}
|
||||
35
utils/determine-turbo-hit-or-miss.js
vendored
35
utils/determine-turbo-hit-or-miss.js
vendored
@@ -1,30 +1,35 @@
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
async function main() {
|
||||
const rootDir = path.join(__dirname, '..');
|
||||
const turboRunDir = path.join(rootDir, '.turbo/runs');
|
||||
async function main(turboRunDirectory, turboRunDirectoryParent) {
|
||||
const turboRunDir = path.join(turboRunDirectoryParent, turboRunDirectory);
|
||||
const turboRunFiles = await fs.readdir(turboRunDir);
|
||||
|
||||
turboRunFiles.forEach(async fileName => {
|
||||
let missCount = 0;
|
||||
|
||||
await Promise.all(
|
||||
turboRunFiles.map(async fileName => {
|
||||
const runFile = path.join(turboRunDir, fileName);
|
||||
const runData = await fs.readJson(runFile);
|
||||
const tasksReports = runData.tasks || [];
|
||||
const { attempted, cached } = runData.execution;
|
||||
|
||||
const missCount = tasksReports.reduce((total, taskData) => {
|
||||
if (taskData.cache.status === 'MISS') {
|
||||
return total + 1;
|
||||
}
|
||||
|
||||
return total;
|
||||
}, 0);
|
||||
missCount += attempted - cached;
|
||||
})
|
||||
);
|
||||
|
||||
// log because STDOUT is how GitHub Actions communicates
|
||||
console.log(missCount);
|
||||
return process.exit(missCount);
|
||||
});
|
||||
|
||||
// Return so we can unit test.
|
||||
return missCount;
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
const turboRunDirectory = '.turbo/runs';
|
||||
const turboRunDirectoryParent = path.join(__dirname, '..');
|
||||
|
||||
main(turboRunDirectory, turboRunDirectoryParent).catch(err => {
|
||||
console.log('error determining Turbo HIT or MISS', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
module.exports = main;
|
||||
|
||||
Reference in New Issue
Block a user