[next]: Fix RSC rewrite behavior (#10415)

- Removes some of the hacks from #10388 that were attempting to resolve an issue with RSC prefetches to `pages` routes in favor of adding rsc rewrites for all dynamic paths, and letting it fall through to a 404 if there's no match
- Fixes an issue where RSC requests were matching the wrong path (filesystem rather than RSC variant) introduced in above mentioned change
  - Closes https://github.com/vercel/next.js/issues/54698

.changeset/long-dingos-punch.md

@@ -0,0 +1,5 @@
+---
+"@vercel/next": patch
+---
+
+Fix RSC rewrite behavior

packages/next/src/server-build.ts

@@ -1074,8 +1074,7 @@ export async function serverBuild({
     canUsePreviewMode,
     prerenderManifest.bypassToken || '',
     true,
-    middleware.dynamicRouteMap,
+    middleware.dynamicRouteMap
-    inversedAppPathManifest
   ).then(arr =>
     localizeDynamicRoutes(
       arr,
@@ -1090,6 +1089,32 @@ export async function serverBuild({
     )
   );
 
+  const pagesPlaceholderRscEntries: Record<string, FileBlob> = {};
+
+  if (appDir) {
+    // since we attempt to rewrite all paths to an .rsc variant,
+    // we need to create dummy rsc outputs for all pages entries
+    // this is so that an RSC request to a `pages` entry will match
+    // rather than falling back to a catchall `app` entry
+    // on the nextjs side, invalid RSC response payloads will correctly trigger an mpa navigation
+    const pagesManifest = path.join(
+      entryPath,
+      outputDirectory,
+      `server/pages-manifest.json`
+    );
+
+    const pagesData = await fs.readJSON(pagesManifest);
+    const pagesEntries = Object.keys(pagesData);
+
+    for (const page of pagesEntries) {
+      const pathName = page.startsWith('/') ? page.slice(1) : page;
+      pagesPlaceholderRscEntries[`${pathName}.rsc`] = new FileBlob({
+        data: '{}',
+        contentType: 'application/json',
+      });
+    }
+  }
+
   const { staticFiles, publicDirectoryFiles, staticDirectoryFiles } =
     await getStaticFiles(entryPath, entryDirectory, outputDirectory);
 
@@ -1249,6 +1274,7 @@ export async function serverBuild({
       ...publicDirectoryFiles,
       ...lambdas,
       ...appRscPrefetches,
+      ...pagesPlaceholderRscEntries,
       // Prerenders may override Lambdas -- this is an intentional behavior.
       ...prerenders,
       ...staticPages,
@@ -1632,72 +1658,22 @@ export async function serverBuild({
           ]
         : []),
 
-      ...(appDir
-        ? [
-            // check routes that end in `.rsc` to see if a page with the resulting name (sans-.rsc) exists in the filesystem
-            // if so, we want to match that page instead. (This matters when prefetching a pages route while on an appdir route)
-            {
-              src: `^${path.posix.join('/', entryDirectory, '/(.*)\\.rsc$')}`,
-              dest: path.posix.join('/', entryDirectory, '/$1'),
-              has: [
-                {
-                  type: 'header',
-                  key: rscHeader,
-                },
-              ],
-              ...(rscPrefetchHeader
-                ? {
-                    missing: [
-                      {
-                        type: 'header',
-                        key: rscPrefetchHeader,
-                      },
-                    ],
-                  }
-                : {}),
-              check: true,
-            } as Route,
-          ]
-        : []),
-
       // These need to come before handle: miss or else they are grouped
       // with that routing section
       ...afterFilesRewrites,
 
-      ...(appDir
-        ? [
-            // rewrite route back to `.rsc`, but skip checking fs
-            {
-              src: `^${path.posix.join(
-                '/',
-                entryDirectory,
-                '/((?!.+\\.rsc).+?)(?:/)?$'
-              )}`,
-              has: [
-                {
-                  type: 'header',
-                  key: rscHeader,
-                },
-              ],
-              dest: path.posix.join('/', entryDirectory, '/$1.rsc'),
-              headers: { vary: rscVaryHeader },
-              continue: true,
-              override: true,
-            },
-          ]
-        : []),
-
-      // make sure 404 page is used when a directory is matched without
-      // an index page
       { handle: 'resource' },
 
       ...fallbackRewrites,
 
+      // make sure 404 page is used when a directory is matched without
+      // an index page
       { src: path.posix.join('/', entryDirectory, '.*'), status: 404 },
 
+      { handle: 'miss' },
+
       // We need to make sure to 404 for /_next after handle: miss since
       // handle: miss is called before rewrites and to prevent rewriting /_next
-      { handle: 'miss' },
       {
         src: path.posix.join(
           '/',

packages/next/src/utils.ts

@@ -305,8 +305,7 @@ export async function getDynamicRoutes(
   canUsePreviewMode?: boolean,
   bypassToken?: string,
   isServerMode?: boolean,
-  dynamicMiddlewareRouteMap?: Map<string, RouteWithSrc>,
+  dynamicMiddlewareRouteMap?: Map<string, RouteWithSrc>
-  appPathRoutesManifest?: Record<string, string>
 ): Promise<RouteWithSrc[]> {
   if (routesManifest) {
     switch (routesManifest.version) {
@@ -379,17 +378,15 @@ export async function getDynamicRoutes(
               },
             ];
           }
+          routes.push({
+            ...route,
+            src: route.src.replace(
+              new RegExp(escapeStringRegexp('(?:/)?$')),
+              '(?:\\.rsc)(?:/)?$'
+            ),
+            dest: route.dest?.replace(/($|\?)/, '.rsc$1'),
+          });
 
-          if (appPathRoutesManifest?.[page]) {
-            routes.push({
-              ...route,
-              src: route.src.replace(
-                new RegExp(escapeStringRegexp('(?:/)?$')),
-                '(?:\\.rsc)(?:/)?$'
-              ),
-              dest: route.dest?.replace(/($|\?)/, '.rsc$1'),
-            });
-          }
           routes.push(route);
           continue;
         }

packages/next/test/fixtures/00-app-dir-catchall-pages/app/[lang]/page.js

@@ -1,7 +1,10 @@
+import Link from 'next/link';
+
 export default function Home() {
   return (
     <div>
       <h1>Home</h1>
+      <Link href="/en/about">About</Link>
     </div>
   );
 }

packages/next/test/fixtures/00-app-dir-catchall-pages/probes.json

@@ -16,7 +16,7 @@
       "headers": {
         "RSC": "1"
       },
-      "mustContain": "<html"
+      "bodyMustBe": "{}"
     },
     {
       "path": "/en/foobar",

packages/next/test/fixtures/00-app-dir/app/gsp/[text]/page.js

@@ -0,0 +1,16 @@
+export default async function DynamicPage({ params }) {
+  return (
+    <main>
+      <h1>Dynamic page</h1>
+      <p>Param: {params.text}</p>
+    </main>
+  );
+}
+
+export async function generateStaticParams() {
+  return [
+    {
+      text: 'one',
+    },
+  ];
+}

packages/next/test/fixtures/00-app-dir/pages/[segmentA]/[segmentB]/[segmentC]/index.js

@@ -0,0 +1,9 @@
+export default function Page(props) {
+  return <div>SSRed Page</div>
+}
+
+export async function getServerSideProps() {
+  return {
+    props: {},
+  }
+}

packages/next/test/fixtures/00-app-dir/pages/docs/categories/foo.js

@@ -0,0 +1,3 @@
+export default function Page() {
+  return 'Hello World'
+}

packages/next/test/fixtures/00-app-dir/vercel.json

@@ -47,6 +47,15 @@
       "mustContain": ":",
       "mustNotContain": "<html"
     },
+    {
+      "path": "/docs/categories/foo?_rsc=f7pci",
+      "status": 200,
+      "headers": {
+        "RSC": "1",
+        "Next-Router-Prefetch": "1"
+      },
+      "bodyMustBe": "{}"
+    },
     {
       "path": "/ssg",
       "status": 200,
@@ -235,6 +244,24 @@
       },
       "mustContain": ":{",
       "mustNotContain": "<html"
+    },
+    {
+      "path": "/gsp/one",
+      "status": 200,
+      "headers": {
+        "RSC": "1"
+      },
+      "mustContain": ":{",
+      "mustNotContain": "<html"
+    },
+    {
+      "path": "/gsp/two",
+      "status": 200,
+      "headers": {
+        "RSC": "1"
+      },
+      "mustContain": ":{",
+      "mustNotContain": "<html"
     }
   ]
 }

test/lib/deployment/test-deployment.js

@@ -220,6 +220,16 @@ async function runProbe(probe, deploymentId, deploymentUrl, ctx) {
     hadTest = true;
   }
 
+  if (probe.bodyMustBe) {
+    if (text !== probe.bodyMustBe) {
+      throw new Error(
+        `Fetched page ${probeUrl} does not have an exact body match of ${probe.bodyMustBe}. Content: ${text}`
+      );
+    }
+
+    hadTest = true;
+  }
+
   /**
    * @type Record<string, string[]>
    */