Restore datadog checking with logs not exit codes (#10527)

Restores DataDog flakey test detection. Briefly disabled because the old method of using exit codes would bail the entire workflow.

Taking the Turbo team's suggestion of looking at `runData.execution` values to avoid an extra loop. Now with unit tests!

.changeset/pink-drinks-obey.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/test.yml

@@ -91,11 +91,18 @@ jobs:
           VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
           VERCEL_TEST_REGISTRATION_URL: ${{ secrets.VERCEL_TEST_REGISTRATION_URL }}
           FORCE_COLOR: '1'
+      - name: 'Determing Turbo HIT or MISS'
+        id: turbo-summary
+        shell: bash
+        run: |
+          TURBO_MISS_COUNT=`node utils/determine-turbo-hit-or-miss.js`
+          echo "MISS COUNT: $TURBO_MISS_COUNT"
+          echo "misses=$TURBO_MISS_COUNT" >> $GITHUB_OUTPUT
       - name: fetch ssl certificate after tests (linux, os x)
         if: matrix.runner != 'windows-latest'
         run: echo | openssl s_client -showcerts -servername 'api.vercel.com' -connect 76.76.21.21:443
       - name: 'Upload Test Report to Datadog'
-        if: always()
+        if: ${{ steps['turbo-summary'].outputs.misses != '0' }}
         run: 'npx @datadog/datadog-ci@2.18.1 junit upload --service vercel-cli .junit-reports'
         env:
           DATADOG_API_KEY: ${{secrets.DATADOG_API_KEY_CLI}}

packages/cli/test/unit/commands/link.test.ts

@@ -13,8 +13,8 @@ import { setupTmpDir } from '../../helpers/setup-unit-fixture';
 
 describe('link', () => {
   it('should prompt for link', async () => {
-    const cwd = setupTmpDir();
     const user = useUser();
+    const cwd = setupTmpDir();
     useTeams('team_dummy');
     const { project } = useProject({
       ...defaultProject,

test/utils/determine-turbo-hit-or-miss/hit/sample.json

@@ -0,0 +1,179 @@
+{
+  "id": "2VLeNEmIfljBVdkudEPZbp5EutG",
+  "version": "1",
+  "turboVersion": "1.10.13",
+  "monorepo": true,
+  "globalCacheInputs": {
+    "rootKey": "You don't understand! I coulda had class. I coulda been a contender. I could've been somebody, instead of a bum, which is what I am.",
+    "files": {
+      "package.json": "5e90f8e7c78d58414bab77fcd16f942f07f426e2",
+      "test/lib/deployment/fetch-retry.js": "f979b34fa9b36be13ecf634e97f085b54f158832",
+      "test/lib/deployment/log.js": "6f750a0cb4cbc59e38d087207d4484a06d83ae51",
+      "test/lib/deployment/now-deploy.js": "a023aab600d7d9ba9da20f96ff060c9ab0bec4c5",
+      "test/lib/deployment/retry-bail-by-default.js": "44830d0f6565655b15fbc441784dc34164749d32",
+      "test/lib/deployment/test-deployment.js": "a18b2cdab36143b27b22313c94fbd74b401bba5a",
+      "test/lib/run-build-lambda.js": "85eba9c6f05ef0584a52d74b4646e1bfbc410484",
+      "turbo-cache-key.json": "523ef4a5572612ec58429b46b5f2b28ac3e24b90",
+      "utils/build.mjs": "074efdecb40a4e4cfdc6acec9439e4632ea1dfa0"
+    },
+    "hashOfExternalDependencies": "a340512aef639cd9",
+    "globalDotEnv": null,
+    "environmentVariables": {
+      "specified": {
+        "env": [],
+        "passThroughEnv": null
+      },
+      "configured": [],
+      "inferred": [],
+      "passthrough": null
+    }
+  },
+  "packages": ["@vercel/error-utils"],
+  "envMode": "infer",
+  "frameworkInference": true,
+  "execution": {
+    "command": "turbo run test --filter=@vercel/error-utils -- test/index.test.ts",
+    "repoPath": "",
+    "success": 0,
+    "failed": 0,
+    "cached": 2,
+    "attempted": 2,
+    "startTime": 1694619745822,
+    "endTime": 1694619746625,
+    "exitCode": 0
+  },
+  "tasks": [
+    {
+      "taskId": "@vercel/error-utils#build",
+      "task": "build",
+      "package": "@vercel/error-utils",
+      "hash": "5d8590c998bec75a",
+      "inputs": {
+        "CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
+        "jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
+        "package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
+        "src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
+        "test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
+        "test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
+        "tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
+      },
+      "hashOfExternalDependencies": "1999225753d8a28a",
+      "cache": {
+        "local": false,
+        "remote": true,
+        "status": "HIT",
+        "source": "REMOTE",
+        "timeSaved": 10154
+      },
+      "command": "node ../../utils/build.mjs",
+      "cliArguments": [],
+      "outputs": ["dist/**"],
+      "excludedOutputs": null,
+      "logFile": "packages/error-utils/.turbo/turbo-build.log",
+      "directory": "packages/error-utils",
+      "dependencies": [],
+      "dependents": ["@vercel/error-utils#test"],
+      "resolvedTaskDefinition": {
+        "outputs": ["dist/**"],
+        "cache": true,
+        "dependsOn": ["^build"],
+        "inputs": [],
+        "outputMode": "new-only",
+        "persistent": false,
+        "env": [],
+        "passThroughEnv": null,
+        "dotEnv": null
+      },
+      "expandedOutputs": [
+        "packages/error-utils/.turbo/turbo-build.log",
+        "packages/error-utils/dist",
+        "packages/error-utils/dist/index.d.ts",
+        "packages/error-utils/dist/index.js",
+        "packages/error-utils/dist/index.js.map"
+      ],
+      "framework": "",
+      "envMode": "loose",
+      "environmentVariables": {
+        "specified": {
+          "env": [],
+          "passThroughEnv": null
+        },
+        "configured": [],
+        "inferred": [],
+        "passthrough": null
+      },
+      "dotEnv": null,
+      "execution": {
+        "startTime": 1694619745975,
+        "endTime": 1694619746484,
+        "exitCode": 0
+      }
+    },
+    {
+      "taskId": "@vercel/error-utils#test",
+      "task": "test",
+      "package": "@vercel/error-utils",
+      "hash": "5bb22ba520f1cbe8",
+      "inputs": {
+        "CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
+        "jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
+        "package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
+        "src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
+        "test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
+        "test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
+        "tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
+      },
+      "hashOfExternalDependencies": "1999225753d8a28a",
+      "cache": {
+        "local": false,
+        "remote": true,
+        "status": "HIT",
+        "source": "REMOTE",
+        "timeSaved": 4572
+      },
+      "command": "jest --reporters=default --reporters=jest-junit --coverage --env node --verbose",
+      "cliArguments": ["test/index.test.ts"],
+      "outputs": null,
+      "excludedOutputs": null,
+      "logFile": "packages/error-utils/.turbo/turbo-test.log",
+      "directory": "packages/error-utils",
+      "dependencies": ["@vercel/error-utils#build"],
+      "dependents": [],
+      "resolvedTaskDefinition": {
+        "outputs": [],
+        "cache": true,
+        "dependsOn": ["build"],
+        "inputs": [],
+        "outputMode": "new-only",
+        "persistent": false,
+        "env": [],
+        "passThroughEnv": null,
+        "dotEnv": null
+      },
+      "expandedOutputs": ["packages/error-utils/.turbo/turbo-test.log"],
+      "framework": "",
+      "envMode": "loose",
+      "environmentVariables": {
+        "specified": {
+          "env": [],
+          "passThroughEnv": null
+        },
+        "configured": [],
+        "inferred": [],
+        "passthrough": null
+      },
+      "dotEnv": null,
+      "execution": {
+        "startTime": 1694619746484,
+        "endTime": 1694619746625,
+        "exitCode": 0
+      }
+    }
+  ],
+  "user": "trek",
+  "scm": {
+    "type": "git",
+    "sha": "0d9d05ba49a4cfda6b60c396b9e84abe34a6c8c1",
+    "branch": "10501/merge"
+  }
+}

test/utils/determine-turbo-hit-or-miss/index.test.js

@@ -0,0 +1,13 @@
+const determineTurboHitOrMiss = require('../../../utils/determine-turbo-hit-or-miss');
+
+describe('determineTurboHitOrMiss', () => {
+  it('detects HITs', async () => {
+    let missCount = await determineTurboHitOrMiss('hit', __dirname);
+    expect(missCount).toEqual(0);
+  });
+
+  it('detects MISSes', async () => {
+    let missCount = await determineTurboHitOrMiss('miss', __dirname);
+    expect(missCount).toEqual(1);
+  });
+});

test/utils/determine-turbo-hit-or-miss/miss/sample.json

@@ -0,0 +1,179 @@
+{
+  "id": "2VLeNEmIfljBVdkudEPZbp5EutG",
+  "version": "1",
+  "turboVersion": "1.10.13",
+  "monorepo": true,
+  "globalCacheInputs": {
+    "rootKey": "You don't understand! I coulda had class. I coulda been a contender. I could've been somebody, instead of a bum, which is what I am.",
+    "files": {
+      "package.json": "5e90f8e7c78d58414bab77fcd16f942f07f426e2",
+      "test/lib/deployment/fetch-retry.js": "f979b34fa9b36be13ecf634e97f085b54f158832",
+      "test/lib/deployment/log.js": "6f750a0cb4cbc59e38d087207d4484a06d83ae51",
+      "test/lib/deployment/now-deploy.js": "a023aab600d7d9ba9da20f96ff060c9ab0bec4c5",
+      "test/lib/deployment/retry-bail-by-default.js": "44830d0f6565655b15fbc441784dc34164749d32",
+      "test/lib/deployment/test-deployment.js": "a18b2cdab36143b27b22313c94fbd74b401bba5a",
+      "test/lib/run-build-lambda.js": "85eba9c6f05ef0584a52d74b4646e1bfbc410484",
+      "turbo-cache-key.json": "523ef4a5572612ec58429b46b5f2b28ac3e24b90",
+      "utils/build.mjs": "074efdecb40a4e4cfdc6acec9439e4632ea1dfa0"
+    },
+    "hashOfExternalDependencies": "a340512aef639cd9",
+    "globalDotEnv": null,
+    "environmentVariables": {
+      "specified": {
+        "env": [],
+        "passThroughEnv": null
+      },
+      "configured": [],
+      "inferred": [],
+      "passthrough": null
+    }
+  },
+  "packages": ["@vercel/error-utils"],
+  "envMode": "infer",
+  "frameworkInference": true,
+  "execution": {
+    "command": "turbo run test --filter=@vercel/error-utils -- test/index.test.ts",
+    "repoPath": "",
+    "success": 0,
+    "failed": 0,
+    "cached": 1,
+    "attempted": 2,
+    "startTime": 1694619745822,
+    "endTime": 1694619746625,
+    "exitCode": 0
+  },
+  "tasks": [
+    {
+      "taskId": "@vercel/error-utils#build",
+      "task": "build",
+      "package": "@vercel/error-utils",
+      "hash": "5d8590c998bec75a",
+      "inputs": {
+        "CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
+        "jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
+        "package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
+        "src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
+        "test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
+        "test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
+        "tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
+      },
+      "hashOfExternalDependencies": "1999225753d8a28a",
+      "cache": {
+        "local": false,
+        "remote": true,
+        "status": "MISS",
+        "source": "REMOTE",
+        "timeSaved": 10154
+      },
+      "command": "node ../../utils/build.mjs",
+      "cliArguments": [],
+      "outputs": ["dist/**"],
+      "excludedOutputs": null,
+      "logFile": "packages/error-utils/.turbo/turbo-build.log",
+      "directory": "packages/error-utils",
+      "dependencies": [],
+      "dependents": ["@vercel/error-utils#test"],
+      "resolvedTaskDefinition": {
+        "outputs": ["dist/**"],
+        "cache": true,
+        "dependsOn": ["^build"],
+        "inputs": [],
+        "outputMode": "new-only",
+        "persistent": false,
+        "env": [],
+        "passThroughEnv": null,
+        "dotEnv": null
+      },
+      "expandedOutputs": [
+        "packages/error-utils/.turbo/turbo-build.log",
+        "packages/error-utils/dist",
+        "packages/error-utils/dist/index.d.ts",
+        "packages/error-utils/dist/index.js",
+        "packages/error-utils/dist/index.js.map"
+      ],
+      "framework": "",
+      "envMode": "loose",
+      "environmentVariables": {
+        "specified": {
+          "env": [],
+          "passThroughEnv": null
+        },
+        "configured": [],
+        "inferred": [],
+        "passthrough": null
+      },
+      "dotEnv": null,
+      "execution": {
+        "startTime": 1694619745975,
+        "endTime": 1694619746484,
+        "exitCode": 0
+      }
+    },
+    {
+      "taskId": "@vercel/error-utils#test",
+      "task": "test",
+      "package": "@vercel/error-utils",
+      "hash": "5bb22ba520f1cbe8",
+      "inputs": {
+        "CHANGELOG.md": "bf7445d194d8a0080f7b98b0181e1f0ac4ef0ef3",
+        "jest.config.js": "36fb3c7c6ce1cee41bbfeb93108375e0e0ea94ef",
+        "package.json": "f0510070d5f969e9cc0d9981c85f5428d4b622c2",
+        "src/index.ts": "adb0bba2a3bc0c86ee6d309b70c26e4ba8d7bbc0",
+        "test/index.test.ts": "982b4a5705dacb00beeadd56ee39402573b1bae6",
+        "test/tsconfig.json": "73807c8c4fc89fdf9e6dec514e8f7864c6c2e843",
+        "tsconfig.json": "86fc498a51913576dba15e12f979de678c848273"
+      },
+      "hashOfExternalDependencies": "1999225753d8a28a",
+      "cache": {
+        "local": false,
+        "remote": true,
+        "status": "HIT",
+        "source": "REMOTE",
+        "timeSaved": 4572
+      },
+      "command": "jest --reporters=default --reporters=jest-junit --coverage --env node --verbose",
+      "cliArguments": ["test/index.test.ts"],
+      "outputs": null,
+      "excludedOutputs": null,
+      "logFile": "packages/error-utils/.turbo/turbo-test.log",
+      "directory": "packages/error-utils",
+      "dependencies": ["@vercel/error-utils#build"],
+      "dependents": [],
+      "resolvedTaskDefinition": {
+        "outputs": [],
+        "cache": true,
+        "dependsOn": ["build"],
+        "inputs": [],
+        "outputMode": "new-only",
+        "persistent": false,
+        "env": [],
+        "passThroughEnv": null,
+        "dotEnv": null
+      },
+      "expandedOutputs": ["packages/error-utils/.turbo/turbo-test.log"],
+      "framework": "",
+      "envMode": "loose",
+      "environmentVariables": {
+        "specified": {
+          "env": [],
+          "passThroughEnv": null
+        },
+        "configured": [],
+        "inferred": [],
+        "passthrough": null
+      },
+      "dotEnv": null,
+      "execution": {
+        "startTime": 1694619746484,
+        "endTime": 1694619746625,
+        "exitCode": 0
+      }
+    }
+  ],
+  "user": "trek",
+  "scm": {
+    "type": "git",
+    "sha": "0d9d05ba49a4cfda6b60c396b9e84abe34a6c8c1",
+    "branch": "10501/merge"
+  }
+}

utils/determine-turbo-hit-or-miss.js

@@ -1,30 +1,35 @@
 const fs = require('fs-extra');
 const path = require('path');
 
-async function main() {
+async function main(turboRunDirectory, turboRunDirectoryParent) {
-  const rootDir = path.join(__dirname, '..');
+  const turboRunDir = path.join(turboRunDirectoryParent, turboRunDirectory);
-  const turboRunDir = path.join(rootDir, '.turbo/runs');
   const turboRunFiles = await fs.readdir(turboRunDir);
 
-  turboRunFiles.forEach(async fileName => {
+  let missCount = 0;
+
+  await Promise.all(
+    turboRunFiles.map(async fileName => {
       const runFile = path.join(turboRunDir, fileName);
       const runData = await fs.readJson(runFile);
-    const tasksReports = runData.tasks || [];
+      const { attempted, cached } = runData.execution;
 
-    const missCount = tasksReports.reduce((total, taskData) => {
+      missCount += attempted - cached;
-      if (taskData.cache.status === 'MISS') {
+    })
-        return total + 1;
+  );
-      }
-
-      return total;
-    }, 0);
 
+  // log because STDOUT is how GitHub Actions communicates
   console.log(missCount);
-    return process.exit(missCount);
+
-  });
+  // Return so we can unit test.
+  return missCount;
 }
 
-main().catch(err => {
+const turboRunDirectory = '.turbo/runs';
+const turboRunDirectoryParent = path.join(__dirname, '..');
+
+main(turboRunDirectory, turboRunDirectoryParent).catch(err => {
   console.log('error determining Turbo HIT or MISS', err);
   process.exit(1);
 });
+
+module.exports = main;