Corrected dependency installation systems (#7088)

* Avoid unnecessary Gemfile installations

* Do not bundle `.output` or `.vercel` into Lambdas

* Use the same input hash format for all CLI Plugins and `vercel build`

* Fixed unit tests

* Fixed the unit tests again

* Fixed the unit tests

* Fixed all the tests

* Exclude useless files

* Consider `.vercelignore` and `.nowignore`

* Fixed error

* Reverted changes

* Deleted useless file

* Fixed tests

* Share input hash format with `vercel-plugin-node`

* Make output inspectable

* Fixed build error

* Extended comment

* Bump Ruby version

* Update Gemfiles

* Update bundles

* Fixed tests

Co-authored-by: Andy Bitz <artzbitz@gmail.com>

packages/build-utils/src/convert-runtime-to-plugin.ts

@@ -5,24 +5,63 @@ import { normalizePath } from './fs/normalize-path';
 import { FILES_SYMBOL, Lambda } from './lambda';
 import type FileBlob from './file-blob';
 import type { BuildOptions, Files } from './types';
+import { getIgnoreFilter } from '.';
 
 /**
  * Convert legacy Runtime to a Plugin.
  * @param buildRuntime - a legacy build() function from a Runtime
+ * @param packageName - the name of the package, for example `vercel-plugin-python`
  * @param ext - the file extension, for example `.py`
  */
 export function convertRuntimeToPlugin(
   buildRuntime: (options: BuildOptions) => Promise<{ output: Lambda }>,
+  packageName: string,
   ext: string
 ) {
   // This `build()` signature should match `plugin.build()` signature in `vercel build`.
   return async function build({ workPath }: { workPath: string }) {
     const opts = { cwd: workPath };
     const files = await glob('**', opts);
-    delete files['vercel.json']; // Builders/Runtimes didn't have vercel.json
-    const entrypoints = await glob(`api/**/*${ext}`, opts);
+
+    // `.output` was already created by the Build Command, so we have
+    // to ensure its contents don't get bundled into the Lambda. Similarily,
+    // we don't want to bundle anything from `.vercel` either. Lastly,
+    // Builders/Runtimes didn't have `vercel.json` or `now.json`.
+    const ignoredPaths = ['.output', '.vercel', 'vercel.json', 'now.json'];
+
+    // We also don't want to provide any files to Runtimes that were ignored
+    // through `.vercelignore` or `.nowignore`, because the Build Step does the same.
+    const ignoreFilter = await getIgnoreFilter(workPath);
+
+    // We're not passing this as an `ignore` filter to the `glob` function above,
+    // so that we can re-use exactly the same `getIgnoreFilter` method that the
+    // Build Step uses (literally the same code).
+    for (const file in files) {
+      const isNative = ignoredPaths.some(item => {
+        return file.startsWith(item);
+      });
+
+      if (isNative || ignoreFilter(file)) {
+        delete files[file];
+      }
+    }
+
+    const entrypointPattern = `api/**/*${ext}`;
+    const entrypoints = await glob(entrypointPattern, opts);
     const pages: { [key: string]: any } = {};
-    const traceDir = join(workPath, '.output', 'runtime-traced-files');
+    const pluginName = packageName.replace('vercel-plugin-', '');
+
+    const traceDir = join(
+      workPath,
+      `.output`,
+      `inputs`,
+      // Legacy Runtimes can only provide API Routes, so that's
+      // why we can use this prefix for all of them. Here, we have to
+      // make sure to not use a cryptic hash name, because people
+      // need to be able to easily inspect the output.
+      `api-routes-${pluginName}`
+    );
+
     await fs.ensureDir(traceDir);
 
     for (const entrypoint of Object.keys(entrypoints)) {

packages/build-utils/src/get-ignore-filter.ts

@@ -0,0 +1,84 @@
+import path from 'path';
+import fs from 'fs-extra';
+import ignore from 'ignore';
+
+interface CodedError extends Error {
+  code: string;
+}
+
+function isCodedError(error: unknown): error is CodedError {
+  return (
+    error !== null &&
+    error !== undefined &&
+    (error as CodedError).code !== undefined
+  );
+}
+
+function clearRelative(s: string) {
+  return s.replace(/(\n|^)\.\//g, '$1');
+}
+
+export default async function (
+  downloadPath: string,
+  rootDirectory?: string | undefined
+) {
+  const readFile = async (p: string) => {
+    try {
+      return await fs.readFile(p, 'utf8');
+    } catch (error: any) {
+      if (
+        error.code === 'ENOENT' ||
+        (error instanceof Error && error.message.includes('ENOENT'))
+      ) {
+        return undefined;
+      }
+
+      throw error;
+    }
+  };
+
+  const vercelIgnorePath = path.join(
+    downloadPath,
+    rootDirectory || '',
+    '.vercelignore'
+  );
+  const nowIgnorePath = path.join(
+    downloadPath,
+    rootDirectory || '',
+    '.nowignore'
+  );
+  const ignoreContents = [];
+
+  try {
+    ignoreContents.push(
+      ...(
+        await Promise.all([readFile(vercelIgnorePath), readFile(nowIgnorePath)])
+      ).filter(Boolean)
+    );
+  } catch (error) {
+    if (isCodedError(error) && error.code === 'ENOTDIR') {
+      console.log(`Warning: Cannot read ignore file from ${vercelIgnorePath}`);
+    } else {
+      throw error;
+    }
+  }
+
+  if (ignoreContents.length === 2) {
+    throw new Error(
+      'Cannot use both a `.vercelignore` and `.nowignore` file. Please delete the `.nowignore` file.'
+    );
+  }
+
+  if (ignoreContents.length === 0) {
+    return () => false;
+  }
+
+  const ignoreFilter: any = ignore().add(clearRelative(ignoreContents[0]!));
+
+  return function (p: string) {
+    // we should not ignore now.json and vercel.json if it asked to.
+    // we depend on these files for building the app with sourceless
+    if (p === 'now.json' || p === 'vercel.json') return false;
+    return ignoreFilter.test(p).ignored;
+  };
+}

packages/build-utils/src/index.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'crypto';
 import FileBlob from './file-blob';
 import FileFsRef from './file-fs-ref';
 import FileRef from './file-ref';
@@ -33,6 +34,7 @@ import { NowBuildError } from './errors';
 import streamToBuffer from './fs/stream-to-buffer';
 import shouldServe from './should-serve';
 import debug from './debug';
+import getIgnoreFilter from './get-ignore-filter';
 
 export {
   FileBlob,
@@ -70,6 +72,7 @@ export {
   isSymbolicLink,
   getLambdaOptionsFromFunction,
   scanParentDirs,
+  getIgnoreFilter,
 };
 
 export {
@@ -132,3 +135,11 @@ export const getPlatformEnv = (name: string): string | undefined => {
   }
   return n;
 };
+
+/**
+ * Helper function for generating file or directories names in `.output/inputs`
+ * for dependencies of files provided to the File System API.
+ */
+export const getInputHash = (source: Buffer | string): string => {
+  return createHash('sha1').update(source).digest('hex');
+};

packages/build-utils/test/unit.convert-runtime-to-plugin.test.ts

@@ -51,14 +51,20 @@ describe('convert-runtime-to-plugin', () => {
 
     const lambdaFiles = await fsToJson(workPath);
     delete lambdaFiles['vercel.json'];
-    const build = await convertRuntimeToPlugin(buildRuntime, '.py');
+
+    const ext = '.py';
+    const packageName = 'vercel-plugin-python';
+    const build = await convertRuntimeToPlugin(buildRuntime, packageName, ext);
 
     await build({ workPath });
 
     const output = await fsToJson(join(workPath, '.output'));
+
     expect(output).toMatchObject({
       'functions-manifest.json': expect.stringContaining('{'),
-      'runtime-traced-files': lambdaFiles,
+      inputs: {
+        'api-routes-python': lambdaFiles,
+      },
       server: {
         pages: {
           api: {
@@ -90,36 +96,35 @@ describe('convert-runtime-to-plugin', () => {
       version: 1,
       files: [
         {
-          input: '../../../../runtime-traced-files/api/db/[id].py',
+          input: `../../../../inputs/api-routes-python/api/db/[id].py`,
           output: 'api/db/[id].py',
         },
         {
-          input: '../../../../runtime-traced-files/api/index.py',
+          input: `../../../../inputs/api-routes-python/api/index.py`,
           output: 'api/index.py',
         },
         {
-          input:
-            '../../../../runtime-traced-files/api/project/[aid]/[bid]/index.py',
+          input: `../../../../inputs/api-routes-python/api/project/[aid]/[bid]/index.py`,
           output: 'api/project/[aid]/[bid]/index.py',
         },
         {
-          input: '../../../../runtime-traced-files/api/users/get.py',
+          input: `../../../../inputs/api-routes-python/api/users/get.py`,
           output: 'api/users/get.py',
         },
         {
-          input: '../../../../runtime-traced-files/api/users/post.py',
+          input: `../../../../inputs/api-routes-python/api/users/post.py`,
           output: 'api/users/post.py',
         },
         {
-          input: '../../../../runtime-traced-files/file.txt',
+          input: `../../../../inputs/api-routes-python/file.txt`,
           output: 'file.txt',
         },
         {
-          input: '../../../../runtime-traced-files/util/date.py',
+          input: `../../../../inputs/api-routes-python/util/date.py`,
           output: 'util/date.py',
         },
         {
-          input: '../../../../runtime-traced-files/util/math.py',
+          input: `../../../../inputs/api-routes-python/util/math.py`,
           output: 'util/math.py',
         },
       ],
@@ -132,36 +137,35 @@ describe('convert-runtime-to-plugin', () => {
       version: 1,
       files: [
         {
-          input: '../../../../../runtime-traced-files/api/db/[id].py',
+          input: `../../../../../inputs/api-routes-python/api/db/[id].py`,
           output: 'api/db/[id].py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/index.py',
+          input: `../../../../../inputs/api-routes-python/api/index.py`,
           output: 'api/index.py',
         },
         {
-          input:
-            '../../../../../runtime-traced-files/api/project/[aid]/[bid]/index.py',
+          input: `../../../../../inputs/api-routes-python/api/project/[aid]/[bid]/index.py`,
           output: 'api/project/[aid]/[bid]/index.py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/users/get.py',
+          input: `../../../../../inputs/api-routes-python/api/users/get.py`,
           output: 'api/users/get.py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/users/post.py',
+          input: `../../../../../inputs/api-routes-python/api/users/post.py`,
           output: 'api/users/post.py',
         },
         {
-          input: '../../../../../runtime-traced-files/file.txt',
+          input: `../../../../../inputs/api-routes-python/file.txt`,
           output: 'file.txt',
         },
         {
-          input: '../../../../../runtime-traced-files/util/date.py',
+          input: `../../../../../inputs/api-routes-python/util/date.py`,
           output: 'util/date.py',
         },
         {
-          input: '../../../../../runtime-traced-files/util/math.py',
+          input: `../../../../../inputs/api-routes-python/util/math.py`,
           output: 'util/math.py',
         },
       ],
@@ -174,36 +178,35 @@ describe('convert-runtime-to-plugin', () => {
       version: 1,
       files: [
         {
-          input: '../../../../../runtime-traced-files/api/db/[id].py',
+          input: `../../../../../inputs/api-routes-python/api/db/[id].py`,
           output: 'api/db/[id].py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/index.py',
+          input: `../../../../../inputs/api-routes-python/api/index.py`,
           output: 'api/index.py',
         },
         {
-          input:
-            '../../../../../runtime-traced-files/api/project/[aid]/[bid]/index.py',
+          input: `../../../../../inputs/api-routes-python/api/project/[aid]/[bid]/index.py`,
           output: 'api/project/[aid]/[bid]/index.py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/users/get.py',
+          input: `../../../../../inputs/api-routes-python/api/users/get.py`,
           output: 'api/users/get.py',
         },
         {
-          input: '../../../../../runtime-traced-files/api/users/post.py',
+          input: `../../../../../inputs/api-routes-python/api/users/post.py`,
           output: 'api/users/post.py',
         },
         {
-          input: '../../../../../runtime-traced-files/file.txt',
+          input: `../../../../../inputs/api-routes-python/file.txt`,
           output: 'file.txt',
         },
         {
-          input: '../../../../../runtime-traced-files/util/date.py',
+          input: `../../../../../inputs/api-routes-python/util/date.py`,
           output: 'util/date.py',
         },
         {
-          input: '../../../../../runtime-traced-files/util/math.py',
+          input: `../../../../../inputs/api-routes-python/util/math.py`,
           output: 'util/math.py',
         },
       ],

packages/plugin-go/src/index.ts

@@ -1,6 +1,7 @@
 import { convertRuntimeToPlugin } from '@vercel/build-utils';
 import * as go from '@vercel/go';
+import { name } from '../package.json';
 
-export const build = convertRuntimeToPlugin(go.build, '.go');
+export const build = convertRuntimeToPlugin(go.build, name, '.go');
 
 export const startDevServer = go.startDevServer;

packages/plugin-go/tsconfig.json

@@ -12,6 +12,7 @@
     "noUnusedParameters": true,
     "outDir": "dist",
     "strict": true,
-    "target": "esnext"
+    "target": "esnext",
+    "resolveJsonModule": true
   }
 }

packages/plugin-node/src/index.ts

@@ -40,6 +40,7 @@ import {
   walkParentDirs,
   normalizePath,
   runPackageJsonScript,
+  getInputHash,
 } from '@vercel/build-utils';
 import { FromSchema } from 'json-schema-to-ts';
 import { getConfig, BaseFunctionConfigSchema } from '@vercel/static-config';
@@ -47,7 +48,6 @@ import { AbortController } from 'abort-controller';
 import { Register, register } from './typescript';
 import { pageToRoute } from './router/page-to-route';
 import { isDynamicRoute } from './router/is-dynamic';
-import crypto from 'crypto';
 
 export { shouldServe };
 export {
@@ -428,10 +428,7 @@ export async function buildEntrypoint({
   installedPaths?: Set<string>;
 }) {
   // Unique hash that will be used as directory name for `.output`.
-  const entrypointHash = crypto
-    .createHash('sha256')
-    .update(entrypoint)
-    .digest('hex');
+  const entrypointHash = 'api-routes-node-' + getInputHash(entrypoint);
   const outputDirPath = join(workPath, '.output');
 
   const { dir, name } = parsePath(entrypoint);

packages/plugin-python/src/index.ts

@@ -1,6 +1,7 @@
 import { convertRuntimeToPlugin } from '@vercel/build-utils';
 import * as python from '@vercel/python';
+import { name } from '../package.json';
 
-export const build = convertRuntimeToPlugin(python.build, '.py');
+export const build = convertRuntimeToPlugin(python.build, name, '.py');
 
 //export const startDevServer = python.startDevServer;

packages/plugin-python/tsconfig.json

@@ -13,6 +13,7 @@
     "outDir": "dist",
     "types": ["node"],
     "strict": true,
-    "target": "esnext"
+    "target": "esnext",
+    "resolveJsonModule": true
   }
 }

packages/plugin-ruby/src/index.ts

@@ -1,6 +1,7 @@
 import { convertRuntimeToPlugin } from '@vercel/build-utils';
 import * as ruby from '@vercel/ruby';
+import { name } from '../package.json';
 
-export const build = convertRuntimeToPlugin(ruby.build, '.rb');
+export const build = convertRuntimeToPlugin(ruby.build, name, '.rb');
 
 //export const startDevServer = ruby.startDevServer;

packages/plugin-ruby/tsconfig.json

@@ -13,6 +13,7 @@
     "outDir": "dist",
     "types": ["node"],
     "strict": true,
-    "target": "esnext"
+    "target": "esnext",
+    "resolveJsonModule": true
   }
 }

packages/ruby/install-ruby.ts

@@ -61,18 +61,27 @@ function getRubyPath(meta: Meta, gemfileContents: string) {
 // process.env.GEM_HOME), and returns
 // the absolute path to it
 export async function installBundler(meta: Meta, gemfileContents: string) {
+  const { gemHome, rubyPath, gemPath, vendorPath, runtime } = getRubyPath(
+    meta,
+    gemfileContents
+  );
+
   // If the new File System API is used (`avoidTopLevelInstall`), the Install Command
   // will have already installed the dependencies, so we don't need to do it again.
   if (meta.avoidTopLevelInstall) {
     debug(
       `Skipping bundler installation, already installed by Install Command`
     );
-  }
 
-  const { gemHome, rubyPath, gemPath, vendorPath, runtime } = getRubyPath(
-    meta,
-    gemfileContents
-  );
+    return {
+      gemHome,
+      rubyPath,
+      gemPath,
+      vendorPath,
+      runtime,
+      bundlerPath: join(gemHome, 'bin', 'bundler'),
+    };
+  }
 
   debug('installing bundler...');
   await execa(gemPath, ['install', 'bundler', '--no-document'], {

packages/ruby/test/fixtures/02-cowsay-vendored/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

packages/ruby/test/fixtures/02-cowsay-vendored/Gemfile

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-ruby "~> 2.5.0"
+ruby "~> 2.7.0"
 
 gem "cowsay", "~> 0.3.0"

packages/ruby/test/fixtures/02-cowsay-vendored/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cowsay (0.3.0)
+
+PLATFORMS
+  x86_64-darwin-21
+
+DEPENDENCIES
+  cowsay (~> 0.3.0)
+
+RUBY VERSION
+   ruby 2.7.5p203
+
+BUNDLED WITH
+   2.2.22

packages/ruby/test/fixtures/02-cowsay-vendored/now.json

@@ -1,6 +1,6 @@
 {
   "version": 2,
   "builds": [{ "src": "index.rb", "use": "@vercel/ruby" }],
-  "build": { "env": { "RUBY_VERSION": "2.5.x" } },
+  "build": { "env": { "RUBY_VERSION": "2.7.x" } },
   "probes": [{ "path": "/", "mustContain": "gem:RANDOMNESS_PLACEHOLDER" }]
 }

packages/ruby/test/fixtures/02-cowsay-vendored/vendor/bundle/ruby/2.7.0/specifications/cowsay-0.3.0.gemspec

@@ -14,19 +14,17 @@ Gem::Specification.new do |s|
   s.executables = ["cowsay".freeze]
   s.files = ["bin/cowsay".freeze]
   s.homepage = "https://github.com/moneydesktop/cowsay".freeze
-  s.rubygems_version = "3.0.3".freeze
+  s.rubygems_version = "3.2.22".freeze
   s.summary = "ASCII art avatars emote your messages".freeze
 
-  s.installed_by_version = "3.0.3" if s.respond_to? :installed_by_version
+  s.installed_by_version = "3.2.22" if s.respond_to? :installed_by_version
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
+  end
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rake>.freeze, [">= 0"])
-    else
-      s.add_dependency(%q<rake>.freeze, [">= 0"])
-    end
+  if s.respond_to? :add_runtime_dependency then
+    s.add_development_dependency(%q<rake>.freeze, [">= 0"])
   else
     s.add_dependency(%q<rake>.freeze, [">= 0"])
   end

packages/ruby/test/fixtures/04-cowsay-vendored-nested/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

packages/ruby/test/fixtures/04-cowsay-vendored-nested/now.json

@@ -1,7 +1,7 @@
 {
   "version": 2,
   "builds": [{ "src": "project/index.rb", "use": "@vercel/ruby" }],
-  "build": { "env": { "RUBY_VERSION": "2.5.x" } },
+  "build": { "env": { "RUBY_VERSION": "2.7.x" } },
   "probes": [
     { "path": "/project/", "mustContain": "gem:RANDOMNESS_PLACEHOLDER" }
   ]

packages/ruby/test/fixtures/04-cowsay-vendored-nested/project/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

packages/ruby/test/fixtures/04-cowsay-vendored-nested/project/Gemfile

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-ruby "~> 2.5.0"
+ruby "~> 2.7.0"
 
 gem "cowsay", "~> 0.3.0"

packages/ruby/test/fixtures/04-cowsay-vendored-nested/project/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cowsay (0.3.0)
+
+PLATFORMS
+  x86_64-darwin-21
+
+DEPENDENCIES
+  cowsay (~> 0.3.0)
+
+RUBY VERSION
+   ruby 2.7.5p203
+
+BUNDLED WITH
+   2.2.22

packages/ruby/test/fixtures/04-cowsay-vendored-nested/project/vendor/bundle/ruby/2.7.0/specifications/cowsay-0.3.0.gemspec

@@ -14,19 +14,17 @@ Gem::Specification.new do |s|
   s.executables = ["cowsay".freeze]
   s.files = ["bin/cowsay".freeze]
   s.homepage = "https://github.com/moneydesktop/cowsay".freeze
-  s.rubygems_version = "3.0.3".freeze
+  s.rubygems_version = "3.2.22".freeze
   s.summary = "ASCII art avatars emote your messages".freeze
 
-  s.installed_by_version = "3.0.3" if s.respond_to? :installed_by_version
+  s.installed_by_version = "3.2.22" if s.respond_to? :installed_by_version
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
+  end
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rake>.freeze, [">= 0"])
-    else
-      s.add_dependency(%q<rake>.freeze, [">= 0"])
-    end
+  if s.respond_to? :add_runtime_dependency then
+    s.add_development_dependency(%q<rake>.freeze, [">= 0"])
   else
     s.add_dependency(%q<rake>.freeze, [">= 0"])
   end

packages/ruby/test/fixtures/05-sinatra/Gemfile

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-ruby "~> 2.5.0"
+ruby "~> 2.7.0"
 
 gem "cowsay", "~> 0.3.0"
 

packages/ruby/test/fixtures/05-sinatra/now.json

@@ -1,6 +1,6 @@
 {
   "version": 2,
   "builds": [{ "src": "index.ru", "use": "@vercel/ruby" }],
-  "build": { "env": { "RUBY_VERSION": "2.5.x" } },
+  "build": { "env": { "RUBY_VERSION": "2.7.x" } },
   "probes": [{ "path": "/", "mustContain": "gem:RANDOMNESS_PLACEHOLDER" }]
 }