[now dev] Bundle canary builders for Now CLI canary (#2661)

* [now dev] Bundle `canary` builders for Now CLI canary

Closes #2641.

* Add unit tests

* More unit tests

* Use `semver.parse()` in `getDistTag()`

* Convert build script to TypeScript, DRY the `getDistTag()` function

* Prettier

package.json

@@ -14,8 +14,8 @@
     "test-lint": "eslint . --ext .js,.ts",
     "prepublishOnly": "yarn build",
     "coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov",
-    "build": "node ./scripts/build.js",
-    "build-dev": "node ./scripts/build.js --dev",
+    "build": "ts-node ./scripts/build.ts",
+    "build-dev": "ts-node ./scripts/build.ts --dev",
     "format-modified": "prettier --parser typescript --write --single-quote `git diff --name-only | grep -e \".*\\.ts$\" -e \".*\\.js$\" | xargs echo`"
   },
   "nyc": {

scripts/build.ts

@@ -1,25 +1,30 @@
-const cpy = require('cpy');
-const tar = require('tar-fs');
-const execa = require('execa');
-const { join } = require('path');
-const pipe = require('promisepipe');
-const { createGzip } = require('zlib');
-const {
+import cpy from 'cpy';
+import tar from 'tar-fs';
+import execa from 'execa';
+import semver from 'semver';
+import { join } from 'path';
+import pipe from 'promisepipe';
+import { createGzip } from 'zlib';
+import {
   createReadStream,
   createWriteStream,
   mkdirp,
   remove,
   writeFile,
   writeJSON
-} = require('fs-extra');
-const pkg = require('../package.json');
+} from 'fs-extra';
+
+import { getDistTag } from '../src/util/dev/builder-cache';
+import pkg from '../package.json';
 
 const dirRoot = join(__dirname, '..');
 
+const bundledBuilders = Object.keys(pkg.devDependencies)
+  .filter(d => d.startsWith('@now/'));
+
 async function createBuildersTarball() {
-  const builders = Object.keys(pkg.devDependencies)
-    .filter(d => d.startsWith('@now/'))
-    .map(d => `${d}@${pkg.devDependencies[d]}`);
+  const distTag = getDistTag(pkg.version);
+  const builders = Array.from(bundledBuilders).map(b => `${b}@${distTag}`);
   console.log(`Creating builders tarball with: ${builders.join(', ')}`);
 
   const buildersDir = join(dirRoot, '.builders');
@@ -39,14 +44,17 @@ async function createBuildersTarball() {
   }
 
   const yarn = join(dirRoot, 'node_modules/yarn/bin/yarn.js');
-  await execa(
-    process.execPath,
-    [ yarn, 'add', '--no-lockfile', ...builders ],
-    { cwd: buildersDir, stdio: 'inherit' }
-  );
+  await execa(process.execPath, [yarn, 'add', '--no-lockfile', ...builders], {
+    cwd: buildersDir,
+    stdio: 'inherit'
+  });
 
   const packer = tar.pack(buildersDir);
-  await pipe(packer, createGzip(), createWriteStream(buildersTarballPath));
+  await pipe(
+    packer,
+    createGzip(),
+    createWriteStream(buildersTarballPath)
+  );
 }
 
 async function main() {
@@ -65,7 +73,7 @@ async function main() {
   // Do the initial `ncc` build
   const src = join(dirRoot, 'src');
   const ncc = join(dirRoot, 'node_modules/@zeit/ncc/dist/ncc/cli.js');
-  const args = [ ncc, 'build', '--source-map' ];
+  const args = [ncc, 'build', '--source-map'];
   if (!isDev) {
     args.push('--minify');
   }

scripts/preinstall.js

@@ -12,10 +12,10 @@ function cmd(command) {
 
 function error(command) {
   console.error('> Error!', command);
-};
+}
 
 function debug(str) {
-  if (process.argv.find((str) => str === '--debug')) {
+  if (process.argv.find(str => str === '--debug')) {
     console.log(`[debug] [${new Date().toISOString()}]`, str);
   }
 }
@@ -68,7 +68,7 @@ async function main() {
     if ((await isBinary(nowPath)) === false) {
       debug(
         'Found file or directory named now but will not delete, ' +
-        'as it seems unrelated to Now CLI'
+          'as it seems unrelated to Now CLI'
       );
       return;
     }
@@ -80,13 +80,15 @@ async function main() {
     if (process.platform !== 'win32') {
       error(
         `An error occured while removing the previous Now CLI installation.\n` +
-        `Please use the this command to remove it: ${cmd(`sudo rm ${nowPath}`)}.\n` +
-        `Then try to install it again.`
+          `Please use the this command to remove it: ${cmd(
+            `sudo rm ${nowPath}`
+          )}.\n` +
+          `Then try to install it again.`
       );
     } else {
       error(
         `An error occured while removing the previous Now CLI installation.\n` +
-        `Please remove ${cmd(nowPath)} manually and try to install it again.`
+          `Please remove ${cmd(nowPath)} manually and try to install it again.`
       );
     }
 
@@ -94,5 +96,4 @@ async function main() {
   }
 }
 
-main()
-  .then(() => process.exit(0));
+main().then(() => process.exit(0));

src/util/dev/builder-cache.ts

@@ -17,6 +17,7 @@ import {
   writeFile,
   remove
 } from 'fs-extra';
+import pkg from '../../../package.json';
 
 import { NoBuilderCacheError, BuilderCacheCleanError } from '../errors-ts';
 import wait from '../output/wait';
@@ -40,6 +41,8 @@ const bundledBuilders = Object.keys(devDependencies).filter(d =>
   d.startsWith('@now/')
 );
 
+const distTag = getDistTag(pkg.version);
+
 export const cacheDirPromise = prepareCacheDir();
 export const builderDirPromise = prepareBuilderDir();
 export const builderModulePathPromise = prepareBuilderModulePath();
@@ -69,6 +72,14 @@ async function readFileOrNull(
   }
 }
 
+export function getDistTag(version: string): string {
+  const parsed = semver.parse(version);
+  if (parsed && typeof parsed.prerelease[0] === 'string') {
+    return parsed.prerelease[0] as string;
+  }
+  return 'latest';
+}
+
 /**
  * Prepare cache directory for installing now-builders
  */
@@ -172,6 +183,40 @@ export function getBuildUtils(packages: string[]): string {
   return `@now/build-utils@${version}`;
 }
 
+export function filterPackage(
+  builderSpec: string,
+  distTag: string,
+  buildersPkg: Package
+) {
+  if (builderSpec in localBuilders) return false;
+  const parsed = npa(builderSpec);
+  if (
+    parsed.name &&
+    parsed.type === 'tag' &&
+    parsed.fetchSpec === distTag &&
+    bundledBuilders.includes(parsed.name) &&
+    buildersPkg.dependencies
+  ) {
+    const parsedInstalled = npa(
+      `${parsed.name}@${buildersPkg.dependencies[parsed.name]}`
+    );
+    if (parsedInstalled.type !== 'version') {
+      return true;
+    }
+    const semverInstalled = semver.parse(parsedInstalled.rawSpec);
+    if (!semverInstalled) {
+      return true;
+    }
+    if (semverInstalled.prerelease.length > 0) {
+      return semverInstalled.prerelease[0] !== distTag;
+    }
+    if (distTag === 'latest') {
+      return false;
+    }
+  }
+  return true;
+}
+
 /**
  * Install a list of builders to the cache directory.
  */
@@ -200,31 +245,9 @@ export async function installBuilders(
   packages.push(getBuildUtils(packages));
 
   // Filter out any packages that come packaged with `now-cli`
-  const packagesToInstall = packages.filter(p => {
-    if (p in localBuilders) return false;
-    const parsed = npa(p);
-    if (!parsed.name) {
-      return true;
-    }
-    if (
-      parsed.type === 'tag' &&
-      parsed.fetchSpec === 'latest' &&
-      bundledBuilders.includes(parsed.name)
-    ) {
-      const parsedInstalled = npa(
-        `${parsed.name}@${buildersPkg.dependencies[parsed.name]}`
-      );
-      if (parsedInstalled.type !== 'version') {
-        return true;
-      }
-      const semverInstalled = semver.parse(parsedInstalled.rawSpec);
-      if (!semverInstalled) {
-        return true;
-      }
-      return semverInstalled.prerelease.length > 0;
-    }
-    return true;
-  });
+  const packagesToInstall = packages.filter(p =>
+    filterPackage(p, distTag, buildersPkg)
+  );
 
   if (packagesToInstall.length === 0) {
     output.debug('No builders need to be installed');

src/util/dev/server.ts

@@ -812,7 +812,7 @@ export default class DevServer {
     const allHeaders = {
       'cache-control': 'public, max-age=0, must-revalidate',
       ...headers,
-      'server': 'now',
+      server: 'now',
       'x-now-trace': 'dev1',
       'x-now-id': nowRequestId,
       'x-now-cache': 'MISS'
@@ -1306,7 +1306,13 @@ export default class DevServer {
   async hasFilesystem(dest: string): Promise<boolean> {
     const requestPath = dest.replace(/^\//, '');
     if (
-      await findBuildMatch(this.buildMatches, this.files, requestPath, this, true)
+      await findBuildMatch(
+        this.buildMatches,
+        this.files,
+        requestPath,
+        this,
+        true
+      )
     ) {
       return true;
     }
@@ -1436,7 +1442,10 @@ async function shouldServe(
     // If there's no `shouldServe()` function, then look up if there's
     // a matching build asset on the `match` that has already been built.
     return true;
-  } else if (!isFilesystem && (await findMatchingRoute(match, requestPath, devServer))) {
+  } else if (
+    !isFilesystem &&
+    (await findMatchingRoute(match, requestPath, devServer))
+  ) {
     // If there's no `shouldServe()` function and no matched asset, then look
     // up if there's a matching build route on the `match` that has already
     // been built.

test/dev-builder.unit.js

@@ -0,0 +1,91 @@
+import test from 'ava';
+
+import { filterPackage } from '../src/util/dev/builder-cache';
+
+test('[dev-builder] filter install "latest", cached canary', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1-canary.0'
+    }
+  };
+  const result = filterPackage('@now/build-utils', 'canary', buildersPkg);
+  t.is(result, true);
+});
+
+test('[dev-builder] filter install "canary", cached stable', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1'
+    }
+  };
+  const result = filterPackage(
+    '@now/build-utils@canary',
+    'latest',
+    buildersPkg
+  );
+  t.is(result, true);
+});
+
+test('[dev-builder] filter install "latest", cached stable', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1'
+    }
+  };
+  const result = filterPackage('@now/build-utils', 'latest', buildersPkg);
+  t.is(result, false);
+});
+
+test('[dev-builder] filter install "canary", cached canary', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1-canary.0'
+    }
+  };
+  const result = filterPackage(
+    '@now/build-utils@canary',
+    'canary',
+    buildersPkg
+  );
+  t.is(result, false);
+});
+
+test('[dev-builder] filter install URL, cached stable', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1'
+    }
+  };
+  const result = filterPackage('https://tarball.now.sh', 'latest', buildersPkg);
+  t.is(result, true);
+});
+
+test('[dev-builder] filter install URL, cached canary', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': '0.0.1-canary.0'
+    }
+  };
+  const result = filterPackage('https://tarball.now.sh', 'canary', buildersPkg);
+  t.is(result, true);
+});
+
+test('[dev-builder] filter install "latest", cached URL - stable', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': 'https://tarball.now.sh'
+    }
+  };
+  const result = filterPackage('@now/build-utils', 'latest', buildersPkg);
+  t.is(result, true);
+});
+
+test('[dev-builder] filter install "latest", cached URL - canary', async t => {
+  const buildersPkg = {
+    dependencies: {
+      '@now/build-utils': 'https://tarball.now.sh'
+    }
+  };
+  const result = filterPackage('@now/build-utils', 'canary', buildersPkg);
+  t.is(result, true);
+});

test/dev-server.unit.js

@@ -54,7 +54,9 @@ function validateResponseHeaders(t, res) {
 
 function get(url) {
   return new Promise((resolve, reject) => {
-    request(url, resolve).on('error', reject).end();
+    request(url, resolve)
+      .on('error', reject)
+      .end();
   });
 }
 
@@ -120,7 +122,9 @@ test(
 test(
   '[DevServer] Allow `cache-control` to be overwritten',
   testFixture('now-dev-headers', async (t, server) => {
-    const res = await get(`${server.address}/?name=cache-control&value=immutable`);
+    const res = await get(
+      `${server.address}/?name=cache-control&value=immutable`
+    );
     t.is(res.headers['cache-control'], 'immutable');
   })
 );