[next] Correct output file tracing and limit calculation (#10631)

While investigating build times noticed that our lambda creation times were increasing linearly with the number of pages which is unexpected since there are mostly shared dependencies. After further investigation it seems we were falling back to our legacy manual `nodeFileTrace` calls in the builder when we shouldn't have been.

Also noticed we were still doing the un-necessary reading/calculating for uncompressed lambda sizes which is as discussed previously isn't required and the only limit we need to keep enforcing is the uncompressed size which is a lot less expensive to compute. 

As a further optimization this adds proper usage of our lstat cache/sema where it wasn't previously and proper parallelizing where applicable. 

These changes reduce tracing/lambda creation times by 3-5x in larger applications and can fix edge cases where we weren't leveraging our more accurate traces from the build. 

Before: 

```sh
Traced Next.js server files in: 444.05ms
Created all serverless functions in: 1:36.311 (m:ss.mmm)
Collected static files (public/, static/, .next/static): 241.47ms
```

After: 

```sh
Traced Next.js server files in: 10.4ms
Created all serverless functions in: 43.684s
Collected static files (public/, static/, .next/static): 250.828ms
```

packages/next/src/utils.ts

@@ -9,7 +9,6 @@ import {
   Prerender,
   getLambdaOptionsFromFunction,
   getPlatformEnv,
-  streamToBuffer,
   NowBuildError,
   isSymbolicLink,
   NodejsLambda,
@@ -25,11 +24,9 @@ import type {
   RouteWithSrc,
 } from '@vercel/routing-utils';
 import { Sema } from 'async-sema';
-import crc32 from 'buffer-crc32';
 import fs, { lstat, stat } from 'fs-extra';
 import path from 'path';
 import semver from 'semver';
-import zlib from 'zlib';
 import url from 'url';
 import { createRequire } from 'module';
 import escapeStringRegexp from 'escape-string-regexp';
@@ -44,7 +41,6 @@ import {
   MIB,
   KIB,
   MAX_UNCOMPRESSED_LAMBDA_SIZE,
-  LAMBDA_RESERVED_COMPRESSED_SIZE,
   LAMBDA_RESERVED_UNCOMPRESSED_SIZE,
 } from './constants';
 
@@ -677,35 +673,34 @@ export function getFilesMapFromReasons(
   return parentFilesMap;
 }
 
-export const collectTracedFiles =
-  (
-    baseDir: string,
-    lstatResults: { [key: string]: ReturnType<typeof lstat> },
-    lstatSema: Sema,
-    reasons: NodeFileTraceReasons,
-    files: { [filePath: string]: FileFsRef }
-  ) =>
-  async (file: string) => {
-    const reason = reasons.get(file);
-    if (reason && reason.type.includes('initial')) {
-      // Initial files are manually added to the lambda later
-      return;
-    }
-    const filePath = path.join(baseDir, file);
+export const collectTracedFiles = async (
+  file: string,
+  baseDir: string,
+  lstatResults: { [key: string]: ReturnType<typeof lstat> },
+  lstatSema: Sema,
+  reasons: NodeFileTraceReasons,
+  files: { [filePath: string]: FileFsRef }
+) => {
+  const reason = reasons.get(file);
+  if (reason && reason.type.includes('initial')) {
+    // Initial files are manually added to the lambda later
+    return;
+  }
+  const filePath = path.join(baseDir, file);
 
-    if (!lstatResults[filePath]) {
-      lstatResults[filePath] = lstatSema
-        .acquire()
-        .then(() => lstat(filePath))
-        .finally(() => lstatSema.release());
-    }
-    const { mode } = await lstatResults[filePath];
+  if (!lstatResults[filePath]) {
+    lstatResults[filePath] = lstatSema
+      .acquire()
+      .then(() => lstat(filePath))
+      .finally(() => lstatSema.release());
+  }
+  const { mode } = await lstatResults[filePath];
 
-    files[file] = new FileFsRef({
-      fsPath: path.join(baseDir, file),
-      mode,
-    });
-  };
+  files[file] = new FileFsRef({
+    fsPath: path.join(baseDir, file),
+    mode,
+  });
+};
 
 export const ExperimentalTraceVersion = `9.0.4-canary.1`;
 
@@ -716,8 +711,6 @@ export type PseudoLayer = {
 export type PseudoFile = {
   file: FileFsRef;
   isSymlink: false;
-  crc32: number;
-  compBuffer: Buffer;
   uncompressedSize: number;
 };
 
@@ -727,54 +720,57 @@ export type PseudoSymbolicLink = {
   symlinkTarget: string;
 };
 
-const compressBuffer = (buf: Buffer): Promise<Buffer> => {
-  return new Promise((resolve, reject) => {
-    zlib.deflateRaw(
-      buf,
-      { level: zlib.constants.Z_BEST_COMPRESSION },
-      (err, compBuf) => {
-        if (err) return reject(err);
-        resolve(compBuf);
-      }
-    );
-  });
-};
-
 export type PseudoLayerResult = {
   pseudoLayer: PseudoLayer;
   pseudoLayerBytes: number;
 };
 
-export async function createPseudoLayer(files: {
-  [fileName: string]: FileFsRef;
+export async function createPseudoLayer({
+  files,
+  lstatSema,
+  lstatResults,
+}: {
+  lstatSema: Sema;
+  lstatResults: { [key: string]: ReturnType<typeof lstat> };
+  files: {
+    [fileName: string]: FileFsRef;
+  };
 }): Promise<PseudoLayerResult> {
-  const pseudoLayer: PseudoLayer = {};
   let pseudoLayerBytes = 0;
+  const pseudoLayer: PseudoLayer = {};
+  const fileNames = Object.keys(files);
 
-  for (const fileName of Object.keys(files)) {
-    const file = files[fileName];
+  await Promise.all(
+    fileNames.map(async fileName => {
+      await lstatSema.acquire();
+      const file = files[fileName];
 
-    if (isSymbolicLink(file.mode)) {
-      const symlinkTarget = await fs.readlink(file.fsPath);
-      pseudoLayer[fileName] = {
-        file,
-        isSymlink: true,
-        symlinkTarget,
-      };
-    } else {
-      const origBuffer = await streamToBuffer(file.toStream());
-      const compBuffer = await compressBuffer(origBuffer);
-      pseudoLayerBytes += compBuffer.byteLength;
-      pseudoLayer[fileName] = {
-        file,
-        compBuffer,
-        isSymlink: false,
-        crc32: crc32.unsigned(origBuffer),
-        uncompressedSize: origBuffer.byteLength,
-      };
-    }
-  }
+      if (isSymbolicLink(file.mode)) {
+        const symlinkTarget = await fs.readlink(file.fsPath);
+        pseudoLayer[fileName] = {
+          file,
+          isSymlink: true,
+          symlinkTarget,
+        };
+      } else {
+        if (!lstatResults[file.fsPath]) {
+          lstatResults[file.fsPath] = lstatSema
+            .acquire()
+            .then(() => lstat(file.fsPath))
+            .finally(() => lstatSema.release());
+        }
+        const { size } = await lstatResults[file.fsPath];
 
+        pseudoLayerBytes += size;
+        pseudoLayer[fileName] = {
+          file,
+          isSymlink: false,
+          uncompressedSize: size,
+        };
+      }
+      lstatSema.release();
+    })
+  );
   return { pseudoLayer, pseudoLayerBytes };
 }
 
@@ -784,10 +780,6 @@ interface CreateLambdaFromPseudoLayersOptions extends LambdaOptionsWithFiles {
   nextVersion?: string;
 }
 
-// measured with 1, 2, 5, 10, and `os.cpus().length || 5`
-// and sema(1) produced the best results
-const createLambdaSema = new Sema(1);
-
 export async function createLambdaFromPseudoLayers({
   files: baseFiles,
   layers,
@@ -795,8 +787,6 @@ export async function createLambdaFromPseudoLayers({
   nextVersion,
   ...lambdaOptions
 }: CreateLambdaFromPseudoLayersOptions) {
-  await createLambdaSema.acquire();
-
   const files: Files = {};
   const addedFiles = new Set();
 
@@ -823,8 +813,6 @@ export async function createLambdaFromPseudoLayers({
     addedFiles.add(fileName);
   }
 
-  createLambdaSema.release();
-
   return new NodejsLambda({
     ...lambdaOptions,
     ...(isStreaming
@@ -1421,7 +1409,6 @@ export type LambdaGroup = {
   isApiLambda: boolean;
   pseudoLayer: PseudoLayer;
   pseudoLayerBytes: number;
-  pseudoLayerUncompressedBytes: number;
 };
 
 export async function getPageLambdaGroups({
@@ -1434,8 +1421,6 @@ export async function getPageLambdaGroups({
   compressedPages,
   tracedPseudoLayer,
   initialPseudoLayer,
-  initialPseudoLayerUncompressed,
-  lambdaCompressedByteLimit,
   internalPages,
   pageExtensions,
 }: {
@@ -1454,8 +1439,6 @@ export async function getPageLambdaGroups({
   };
   tracedPseudoLayer: PseudoLayer;
   initialPseudoLayer: PseudoLayerResult;
-  initialPseudoLayerUncompressed: number;
-  lambdaCompressedByteLimit: number;
   internalPages: string[];
   pageExtensions?: string[];
 }) {
@@ -1497,19 +1480,16 @@ export async function getPageLambdaGroups({
         group.isPrerenders === isPrerenderRoute;
 
       if (matches) {
-        let newTracedFilesSize = group.pseudoLayerBytes;
-        let newTracedFilesUncompressedSize = group.pseudoLayerUncompressedBytes;
+        let newTracedFilesUncompressedSize = group.pseudoLayerBytes;
 
         for (const newPage of newPages) {
           Object.keys(pageTraces[newPage] || {}).map(file => {
             if (!group.pseudoLayer[file]) {
               const item = tracedPseudoLayer[file] as PseudoFile;
 
-              newTracedFilesSize += item.compBuffer?.byteLength || 0;
               newTracedFilesUncompressedSize += item.uncompressedSize || 0;
             }
           });
-          newTracedFilesSize += compressedPages[newPage].compBuffer.byteLength;
           newTracedFilesUncompressedSize +=
             compressedPages[newPage].uncompressedSize;
         }
@@ -1517,11 +1497,8 @@ export async function getPageLambdaGroups({
         const underUncompressedLimit =
           newTracedFilesUncompressedSize <
           MAX_UNCOMPRESSED_LAMBDA_SIZE - LAMBDA_RESERVED_UNCOMPRESSED_SIZE;
-        const underCompressedLimit =
-          newTracedFilesSize <
-          lambdaCompressedByteLimit - LAMBDA_RESERVED_COMPRESSED_SIZE;
 
-        return underUncompressedLimit && underCompressedLimit;
+        return underUncompressedLimit;
       }
       return false;
     });
@@ -1535,7 +1512,6 @@ export async function getPageLambdaGroups({
         isPrerenders: isPrerenderRoute,
         isApiLambda: !!isApiPage(page),
         pseudoLayerBytes: initialPseudoLayer.pseudoLayerBytes,
-        pseudoLayerUncompressedBytes: initialPseudoLayerUncompressed,
         pseudoLayer: Object.assign({}, initialPseudoLayer.pseudoLayer),
       };
       groups.push(newGroup);
@@ -1545,21 +1521,16 @@ export async function getPageLambdaGroups({
     for (const newPage of newPages) {
       Object.keys(pageTraces[newPage] || {}).map(file => {
         const pseudoItem = tracedPseudoLayer[file] as PseudoFile;
-        const compressedSize = pseudoItem?.compBuffer?.byteLength || 0;
 
         if (!matchingGroup!.pseudoLayer[file]) {
           matchingGroup!.pseudoLayer[file] = pseudoItem;
-          matchingGroup!.pseudoLayerBytes += compressedSize;
-          matchingGroup!.pseudoLayerUncompressedBytes +=
-            pseudoItem.uncompressedSize || 0;
+          matchingGroup!.pseudoLayerBytes += pseudoItem.uncompressedSize || 0;
         }
       });
 
       // ensure the page file itself is accounted for when grouping as
       // large pages can be created that can push the group over the limit
       matchingGroup!.pseudoLayerBytes +=
-        compressedPages[newPage].compBuffer.byteLength;
-      matchingGroup!.pseudoLayerUncompressedBytes +=
         compressedPages[newPage].uncompressedSize;
     }
   }
@@ -1571,7 +1542,6 @@ export const outputFunctionFileSizeInfo = (
   pages: string[],
   pseudoLayer: PseudoLayer,
   pseudoLayerBytes: number,
-  pseudoLayerUncompressedBytes: number,
   compressedPages: {
     [page: string]: PseudoFile;
   }
@@ -1583,15 +1553,10 @@ export const outputFunctionFileSizeInfo = (
       ', '
     )}`
   );
-  exceededLimitOutput.push([
-    'Large Dependencies',
-    'Uncompressed size',
-    'Compressed size',
-  ]);
+  exceededLimitOutput.push(['Large Dependencies', 'Uncompressed size']);
 
   const dependencies: {
     [key: string]: {
-      compressed: number;
       uncompressed: number;
     };
   } = {};
@@ -1603,19 +1568,16 @@ export const outputFunctionFileSizeInfo = (
 
       if (!dependencies[depKey]) {
         dependencies[depKey] = {
-          compressed: 0,
           uncompressed: 0,
         };
       }
 
-      dependencies[depKey].compressed += fileItem.compBuffer.byteLength;
       dependencies[depKey].uncompressed += fileItem.uncompressedSize;
     }
   }
 
   for (const page of pages) {
     dependencies[`pages/${page}`] = {
-      compressed: compressedPages[page].compBuffer.byteLength,
       uncompressed: compressedPages[page].uncompressedSize,
     };
   }
@@ -1627,10 +1589,10 @@ export const outputFunctionFileSizeInfo = (
       const aDep = dependencies[a];
       const bDep = dependencies[b];
 
-      if (aDep.compressed > bDep.compressed) {
+      if (aDep.uncompressed > bDep.uncompressed) {
         return -1;
       }
-      if (aDep.compressed < bDep.compressed) {
+      if (aDep.uncompressed < bDep.uncompressed) {
         return 1;
       }
       return 0;
@@ -1638,15 +1600,11 @@ export const outputFunctionFileSizeInfo = (
     .forEach(depKey => {
       const dep = dependencies[depKey];
 
-      if (dep.compressed < 100 * KIB && dep.uncompressed < 500 * KIB) {
+      if (dep.uncompressed < 500 * KIB) {
         // ignore smaller dependencies to reduce noise
         return;
       }
-      exceededLimitOutput.push([
-        depKey,
-        prettyBytes(dep.uncompressed),
-        prettyBytes(dep.compressed),
-      ]);
+      exceededLimitOutput.push([depKey, prettyBytes(dep.uncompressed)]);
       numLargeDependencies += 1;
     });
 
@@ -1657,11 +1615,7 @@ export const outputFunctionFileSizeInfo = (
   }
 
   exceededLimitOutput.push([]);
-  exceededLimitOutput.push([
-    'All dependencies',
-    prettyBytes(pseudoLayerUncompressedBytes),
-    prettyBytes(pseudoLayerBytes),
-  ]);
+  exceededLimitOutput.push(['All dependencies', prettyBytes(pseudoLayerBytes)]);
 
   console.log(
     textTable(exceededLimitOutput, {
@@ -1672,13 +1626,11 @@ export const outputFunctionFileSizeInfo = (
 
 export const detectLambdaLimitExceeding = async (
   lambdaGroups: LambdaGroup[],
-  compressedSizeLimit: number,
   compressedPages: {
     [page: string]: PseudoFile;
   }
 ) => {
   // show debug info if within 5 MB of exceeding the limit
-  const COMPRESSED_SIZE_LIMIT_CLOSE = compressedSizeLimit - 5 * MIB;
   const UNCOMPRESSED_SIZE_LIMIT_CLOSE = MAX_UNCOMPRESSED_LAMBDA_SIZE - 5 * MIB;
 
   let numExceededLimit = 0;
@@ -1688,13 +1640,9 @@ export const detectLambdaLimitExceeding = async (
   // pre-iterate to see if we are going to exceed the limit
   // or only get close so our first log line can be correct
   const filteredGroups = lambdaGroups.filter(group => {
-    const exceededLimit =
-      group.pseudoLayerBytes > compressedSizeLimit ||
-      group.pseudoLayerUncompressedBytes > MAX_UNCOMPRESSED_LAMBDA_SIZE;
+    const exceededLimit = group.pseudoLayerBytes > MAX_UNCOMPRESSED_LAMBDA_SIZE;
 
-    const closeToLimit =
-      group.pseudoLayerBytes > COMPRESSED_SIZE_LIMIT_CLOSE ||
-      group.pseudoLayerUncompressedBytes > UNCOMPRESSED_SIZE_LIMIT_CLOSE;
+    const closeToLimit = group.pseudoLayerBytes > UNCOMPRESSED_SIZE_LIMIT_CLOSE;
 
     if (
       closeToLimit ||
@@ -1717,8 +1665,6 @@ export const detectLambdaLimitExceeding = async (
       if (numExceededLimit || numCloseToLimit) {
         console.log(
           `Warning: Max serverless function size of ${prettyBytes(
-            compressedSizeLimit
-          )} compressed or ${prettyBytes(
             MAX_UNCOMPRESSED_LAMBDA_SIZE
           )} uncompressed${numExceededLimit ? '' : ' almost'} reached`
         );
@@ -1732,7 +1678,6 @@ export const detectLambdaLimitExceeding = async (
       group.pages,
       group.pseudoLayer,
       group.pseudoLayerBytes,
-      group.pseudoLayerUncompressedBytes,
       compressedPages
     );
   }
@@ -2567,15 +2512,14 @@ export function normalizeEdgeFunctionPath(
       appPathRoutesManifest[ogRoute] ||
       shortPath.replace(/(^|\/)(page|route)$/, '')
     ).replace(/^\//, '');
-
-    if (!shortPath || shortPath === '/') {
-      shortPath = 'index';
-    }
   }
 
   if (shortPath.startsWith('pages/')) {
     shortPath = shortPath.replace(/^pages\//, '');
   }
+  if (!shortPath || shortPath === '/') {
+    shortPath = 'index';
+  }
 
   return shortPath;
 }