[build-utils] Preserve symlinks for FileRef and FileBlob types in download() (#7534)

Co-authored-by: Nathan Rajlich <n@n8.io>

packages/build-utils/src/file-blob.ts

@@ -48,6 +48,10 @@ export default class FileBlob implements FileBase {
     return new FileBlob({ mode, contentType, data });
   }
 
+  async toStreamAsync(): Promise<NodeJS.ReadableStream> {
+    return this.toStream();
+  }
+
   toStream(): NodeJS.ReadableStream {
     return intoStream(this.data);
   }

packages/build-utils/src/fs/download.ts

@@ -3,6 +3,7 @@ import debug from '../debug';
 import FileFsRef from '../file-fs-ref';
 import { File, Files, Meta } from '../types';
 import { remove, mkdirp, readlink, symlink } from 'fs-extra';
+import streamToBuffer from './stream-to-buffer';
 
 export interface DownloadedFiles {
   [filePath: string]: FileFsRef;
@@ -15,19 +16,44 @@ export function isSymbolicLink(mode: number): boolean {
   return (mode & S_IFMT) === S_IFLNK;
 }
 
+async function prepareSymlinkTarget(
+  file: File,
+  fsPath: string
+): Promise<string> {
+  const mkdirPromise = mkdirp(path.dirname(fsPath));
+  if (file.type === 'FileFsRef') {
+    const [target] = await Promise.all([readlink(file.fsPath), mkdirPromise]);
+    return target;
+  }
+
+  if (file.type === 'FileRef' || file.type === 'FileBlob') {
+    const targetPathBufferPromise = await streamToBuffer(
+      await file.toStreamAsync()
+    );
+    const [targetPathBuffer] = await Promise.all([
+      targetPathBufferPromise,
+      mkdirPromise,
+    ]);
+    return targetPathBuffer.toString('utf8');
+  }
+
+  throw new Error(
+    `file.type "${(file as any).type}" not supported for symlink`
+  );
+}
+
 async function downloadFile(file: File, fsPath: string): Promise<FileFsRef> {
   const { mode } = file;
-  if (mode && isSymbolicLink(mode) && file.type === 'FileFsRef') {
+
-    const [target] = await Promise.all([
+  if (isSymbolicLink(mode)) {
-      readlink(file.fsPath),
+    const target = await prepareSymlinkTarget(file, fsPath);
-      mkdirp(path.dirname(fsPath)),
+
-    ]);
     await symlink(target, fsPath);
     return FileFsRef.fromFsPath({ mode, fsPath });
-  } else {
-    const stream = file.toStream();
-    return FileFsRef.fromStream({ mode, stream, fsPath });
   }
+
+  const stream = file.toStream();
+  return FileFsRef.fromStream({ mode, stream, fsPath });
 }
 
 async function removeFile(basePath: string, fileMatched: string) {

packages/build-utils/test/unit.test.ts

@@ -1,7 +1,7 @@
 import ms from 'ms';
 import path from 'path';
-import fs from 'fs-extra';
+import fs, { readlink } from 'fs-extra';
-import { strict as assert } from 'assert';
+import { strict as assert, strictEqual } from 'assert';
 import { createZip } from '../src/lambda';
 import { getSupportedNodeVersion } from '../src/fs/node-version';
 import download from '../src/fs/download';
@@ -14,6 +14,7 @@ import {
   runNpmInstall,
   runPackageJsonScript,
   scanParentDirs,
+  FileBlob,
 } from '../src';
 
 async function expectBuilderError(promise: Promise<any>, pattern: string) {
@@ -47,7 +48,7 @@ afterEach(() => {
   console.warn = originalConsoleWarn;
 });
 
-it('should re-create symlinks properly', async () => {
+it('should re-create FileFsRef symlinks properly', async () => {
   if (process.platform === 'win32') {
     console.log('Skipping test on windows');
     return;
@@ -69,6 +70,72 @@ it('should re-create symlinks properly', async () => {
   assert(linkStat.isSymbolicLink());
   assert(linkDirStat.isSymbolicLink());
   assert(aStat.isFile());
+
+  const [linkDirContents, linkTextContents] = await Promise.all([
+    readlink(path.join(outDir, 'link-dir')),
+    readlink(path.join(outDir, 'link.txt')),
+  ]);
+
+  strictEqual(linkDirContents, 'dir');
+  strictEqual(linkTextContents, './a.txt');
+});
+
+it('should re-create FileBlob symlinks properly', async () => {
+  if (process.platform === 'win32') {
+    console.log('Skipping test on windows');
+    return;
+  }
+
+  const files = {
+    'a.txt': new FileBlob({
+      mode: 33188,
+      contentType: undefined,
+      data: 'a text',
+    }),
+    'dir/b.txt': new FileBlob({
+      mode: 33188,
+      contentType: undefined,
+      data: 'b text',
+    }),
+    'link-dir': new FileBlob({
+      mode: 41453,
+      contentType: undefined,
+      data: 'dir',
+    }),
+    'link.txt': new FileBlob({
+      mode: 41453,
+      contentType: undefined,
+      data: 'a.txt',
+    }),
+  };
+
+  strictEqual(Object.keys(files).length, 4);
+
+  const outDir = path.join(__dirname, 'symlinks-out');
+  await fs.remove(outDir);
+
+  const files2 = await download(files, outDir);
+  strictEqual(Object.keys(files2).length, 4);
+
+  const [linkStat, linkDirStat, aStat, dirStat] = await Promise.all([
+    fs.lstat(path.join(outDir, 'link.txt')),
+    fs.lstat(path.join(outDir, 'link-dir')),
+    fs.lstat(path.join(outDir, 'a.txt')),
+    fs.lstat(path.join(outDir, 'dir')),
+  ]);
+
+  assert(linkStat.isSymbolicLink());
+  assert(linkDirStat.isSymbolicLink());
+  assert(aStat.isFile());
+  assert(dirStat.isDirectory());
+
+  const [linkDirContents, linkTextContents] = await Promise.all([
+    readlink(path.join(outDir, 'link-dir')),
+    readlink(path.join(outDir, 'link.txt')),
+  ]);
+
+  strictEqual(linkDirContents, 'dir');
+  strictEqual(linkTextContents, 'a.txt');
 });
 
 it('should create zip files with symlinks properly', async () => {