feat: push command (#234)

* feat: wip push command
* chore: rebase to new structure
* chore: handle cli push arguments
* chore: get signed url; upload file to s3; create hash from files;
* chore: upsert logic
* upsert + version logic updated
* chore: multiple sign & upload files
* chore: improve destination arg validation
* chore: execution time
* fix: added isAuthorized client logic
* chore: extended upload files logic
* chore: default branch logic
* fix: naming and other corrections
* fix: destination arg regexp updated
* chore: check the existence of the organization
* fix: file path to s3 upload
* chore: refactor gathering files to upload
* fix: path resolve for root
* chore: add files output
* chore: fix files output
* chore: simplify findConfig
* fix: bundle definition before pushing
* fix: upload all related files to plugins
* fix: ignore file & naming corrections
* fix: bundling definition
* chore: minor cleanup

Co-authored-by: romanhotsiy <gotsijroman@gmail.com>

packages/cli/src/commands/push.ts

@@ -0,0 +1,226 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import fetch from 'node-fetch';
+import { performance } from 'perf_hooks';
+import { yellow, green, blue } from 'colorette';
+import { createHash } from 'crypto';
+import { bundle, Config, loadConfig, RedoclyClient, IGNORE_FILE, BundleOutputFormat } from '@redocly/openapi-core';
+import {
+  promptUser,
+  exitWithError,
+  printExecutionTime,
+  getFallbackEntryPointsOrExit,
+  getTotals,
+  pluralize,
+  dumpBundle,
+} from '../utils';
+
+type Source = {
+  files: string[];
+  branchName?: string;
+  root?: string;
+}
+
+export async function handlePush (argv: {
+  entrypoint?: string;
+  destination?: string;
+  branchName?: string;
+  upsert?: boolean;
+  'run-id'?: string;
+}) {
+
+  const client = new RedoclyClient();
+  const isAuthorized = await client.isAuthorizedWithRedocly();
+  if (!isAuthorized) {
+    const clientToken = await promptUser(
+       green(`\n  🔑 Copy your access token from ${blue('https://app.redoc.ly/profile')} and paste it below`)
+    );
+    await client.login(clientToken);
+  }
+
+  const startedAt = performance.now();
+  const { entrypoint, destination, branchName, upsert } = argv;
+
+  if (!validateDestination(destination!)) {
+    exitWithError(`Destination argument value is not valid, please use the right format: ${yellow('<@organization-id/api-name@api-version>')}`);
+  }
+
+  const [ organizationId, apiName, apiVersion ] = getDestinationProps(destination!);
+  await doesOrganizationExist(organizationId);
+  const { version } = await client.getDefinitionVersion(organizationId, apiName, apiVersion);
+
+  if (!version && !upsert) {
+    exitWithError(`
+  The definition version ${blue(apiName)}/${blue(apiVersion)} does not exist in organization ${blue(organizationId)}!
+  ${yellow('Suggestion:')} please use ${blue('-u')} or ${blue('--upsert')} to create definition.
+    `);
+  }
+
+  if (version) {
+    const { definitionId, defaultBranch, id } = version;
+    const updatePatch = await collectAndUploadFiles(branchName || defaultBranch.name);
+    await client.updateDefinitionVersion(definitionId, id, updatePatch);
+  } else if (upsert) {
+    await doesOrganizationExist(organizationId);
+    const { definition } = await client.getDefinitionByName(apiName, organizationId);
+    let definitionId;
+    if (!definition) {
+      const { def } = await client.createDefinition(organizationId, apiName);
+      definitionId = def.definition.id;
+    } else {
+      definitionId = definition.id;
+    }
+    const updatePatch = await collectAndUploadFiles(branchName || 'main');
+    await client.createDefinitionVersion(definitionId, apiVersion, "FILE", updatePatch.source);
+  }
+
+  process.stderr.write(`Definition: ${blue(entrypoint!)} is successfully pushed to Redocly API Registry \n`);
+  printExecutionTime('push', startedAt, entrypoint!);
+
+  async function doesOrganizationExist(organizationId: string) {
+    const { organizationById } = await client.getOrganizationId(organizationId);
+    if (!organizationById) { exitWithError(`Organization ${blue(organizationId)} not found`); }
+  }
+
+  async function collectAndUploadFiles(branch: string) {
+    let source: Source = { files: [], branchName: branch };
+    const filesToUpload = await collectFilesToUpload(entrypoint!);
+    const filesHash = hashFiles(filesToUpload.files);
+
+    process.stdout.write(`Uploading ${filesToUpload.files.length} ${pluralize('file', filesToUpload.files.length)}:\n`);
+    let uploaded = 0;
+    for (let file of filesToUpload.files) {
+      const { signFileUploadCLI } = await client.getSignedUrl(organizationId, filesHash, file.keyOnS3);
+      const { signedFileUrl, uploadedFilePath } = signFileUploadCLI;
+      if (file.filePath === filesToUpload.root) { source['root'] = uploadedFilePath; }
+      source.files.push(uploadedFilePath);
+      process.stdout.write(`Uploading ${file.contents ? 'bundle for ' : ''}${blue(file.filePath)}...`);
+      await uploadFileToS3(signedFileUrl, file.contents || file.filePath);
+      process.stdout.write(green(`✓ (${++uploaded}/${filesToUpload.files.length})\n`));
+    }
+
+    process.stdout.write('\n');
+    return {
+      sourceType: "FILE",
+      source: JSON.stringify(source)
+    }
+  }
+}
+
+function getFilesList(dir: string, files?: any): string[] {
+  files = files || [];
+  const filesAndDirs = fs.readdirSync(dir);
+  for (const name of filesAndDirs) {
+    if (fs.statSync(path.join(dir, name)).isDirectory()) {
+      files = getFilesList(path.join(dir, name), files);
+    } else {
+      const currentPath = dir + '/' + name;
+      files.push(currentPath);
+    }
+  }
+  return files;
+}
+
+async function collectFilesToUpload(entrypoint: string) {
+  let files: { filePath: string, keyOnS3: string, contents?: Buffer }[] = [];
+  const config: Config = await loadConfig();
+  const entrypoints = await getFallbackEntryPointsOrExit([entrypoint], config);
+  const entrypointPath = entrypoints[0];
+
+  process.stdout.write('Bundling definition\n');
+
+  const {bundle: openapiBundle, problems} = await bundle({
+    config,
+    ref: entrypointPath
+  });
+
+  const fileTotals = getTotals(problems);
+
+  if (fileTotals.errors === 0) {
+    process.stdout.write(
+      `Created a bundle for ${blue(entrypoint)} ${
+        fileTotals.warnings > 0 ? 'with warnings' : ''
+      }\n`
+    );
+  } else {
+    exitWithError(`Failed to create a bundle for ${blue(entrypoint)}\n`)
+  }
+
+  const fileExt = path.extname(entrypointPath).split('.').pop();
+  files.push(getFileEntry(entrypointPath, dumpBundle(openapiBundle.parsed, fileExt as BundleOutputFormat)));
+
+  if (fs.existsSync('package.json')) { files.push(getFileEntry('package.json')); }
+  if (fs.existsSync(IGNORE_FILE)) { files.push(getFileEntry(IGNORE_FILE)); }
+  if (config.configFile) {
+    files.push(getFileEntry(config.configFile));
+    if (config.referenceDocs.htmlTemplate) {
+      const dir = getFolder(config.referenceDocs.htmlTemplate);
+      const fileList = getFilesList(dir, []);
+      files.push(...fileList.map(f => getFileEntry(f)));
+    }
+    if (config.rawConfig && config.rawConfig.lint && config.rawConfig.lint.plugins) {
+      let pluginFiles = new Set<string>();
+      for (const plugin of config.rawConfig.lint.plugins) {
+        if (typeof plugin !== 'string') continue;
+        const fileList = getFilesList(getFolder(plugin), []);
+        fileList.forEach(f => pluginFiles.add(f));
+      }
+      files.push(
+        ...(filterPluginFilesByExt(Array.from(pluginFiles))).map(f => getFileEntry(f))
+      );
+    }
+  }
+  return {
+    files,
+    root: path.resolve(entrypointPath),
+  }
+
+  function filterPluginFilesByExt(files: string[]) {
+    return files.filter((file: string) => {
+      const fileExt = path.extname(file).toLowerCase();
+      return fileExt === '.js' || fileExt === '.ts' || fileExt === '.mjs' || fileExt === 'json';
+    });
+  }
+
+  function getFileEntry(filename: string, contents?: string) {
+    return {
+      filePath: path.resolve(filename),
+      keyOnS3: config.configFile
+        ? path.relative(path.dirname(config.configFile), filename)
+        : path.basename(filename),
+      contents: contents && Buffer.from(contents, 'utf-8') || undefined,
+    }
+  }
+}
+
+function getFolder(filePath: string) {
+  return path.resolve(path.dirname(filePath));
+}
+
+function hashFiles(filePaths: { filePath: string }[]) {
+  let sum = createHash('sha256');
+  filePaths.forEach(file => sum.update(fs.readFileSync(file.filePath)));
+  return sum.digest('hex');
+}
+
+function validateDestination(destination: string) {
+  const regexp = /^@+[a-zA-Z0-9-_]{1,}\/+[a-zA-Z0-9-_ ]{1,}@[a-zA-Z0-9-_ ]{1,}$/g;
+  return regexp.test(destination);
+}
+
+function getDestinationProps(destination: string) {
+  return destination.substring(1).split(/[@\/]/);
+}
+
+function uploadFileToS3(url: string, filePathOrBuffer: string | Buffer) {
+  const fileSizeInBytes = typeof filePathOrBuffer === 'string' ? fs.statSync(filePathOrBuffer).size : filePathOrBuffer.byteLength;
+  let readStream = typeof filePathOrBuffer === 'string' ? fs.createReadStream(filePathOrBuffer) : filePathOrBuffer;
+
+  return fetch(url, {
+    method: 'PUT',
+    headers: {
+      'Content-Length': fileSizeInBytes.toString()
+    },
+    body: readStream
+  })
+}

packages/cli/src/index.ts

@@ -11,6 +11,7 @@ import { previewDocs } from './commands/preview-docs';
 import { handleStats } from './commands/stats';
 import { handleSplit } from './commands/split';
 import { handleJoin } from './commands/join';
+import { handlePush } from './commands/push';
 import { handleLint } from './commands/lint';
 import { handleBundle } from './commands/bundle';
 const version = require('../package.json').version;
@@ -70,6 +71,17 @@ yargs
       }),
     (argv) => { handleJoin(argv, version) }
   )
+  .command('push <entrypoint> <destination> [branchName]', 'Push a API definition to the Redocly API Registry',
+    (yargs) => yargs
+    .positional('entrypoint', { type: 'string' })
+    .positional('destination', { type: 'string' })
+    .positional('branchName', { type: 'string' })
+    .option({
+      'upsert': { type: 'boolean', alias: 'u' },
+      'run-id': { type: 'string', requiresArg: true }
+    }),
+    (argv) => { handlePush(argv) }
+  )
   .command('lint [entrypoints...]', 'Lint definition.',
     (yargs) => yargs
       .positional('entrypoints', { array: true, type: 'string', demandOption: true })

packages/cli/src/utils.ts

@@ -97,7 +97,7 @@ export class CircularJSONNotSupportedError extends Error {
   }
 }
 
-export function dumpBundle(obj: any, format: BundleOutputFormat, dereference?: boolean) {
+export function dumpBundle(obj: any, format: BundleOutputFormat, dereference?: boolean): string {
   if (format === 'json') {
     try {
       return JSON.stringify(obj, null, 2);

packages/core/__tests__/resolve.test.ts

@@ -192,8 +192,7 @@ describe('collect refs', () => {
 
     expect(resolvedRefs).toBeDefined();
 
-    expect(Array.from(resolvedRefs.keys()).map((ref) => ref.substring(cwd.length + 1)))
-      .toMatchInlineSnapshot(`
+    expect(Array.from(resolvedRefs.keys()).map((ref) => ref.substring(cwd.length + 1))).toMatchInlineSnapshot(`
       Array [
         "openapi-with-back.yaml::./schemas/type-a.yaml#/",
         "openapi-with-back.yaml::./schemas/type-b.yaml#/",

packages/core/src/config/config.ts

@@ -24,7 +24,7 @@ import recommended from './recommended';
 import { NodeType } from '../types';
 import { RedoclyClient } from '../redocly';
 
-const IGNORE_FILE = '.redocly.lint-ignore.yaml';
+export const IGNORE_FILE = '.redocly.lint-ignore.yaml';
 const IGNORE_BANNER =
   `# This file instructs Redocly's linter to ignore the rules contained for specific parts of your API.\n` +
   `# See https://redoc.ly/docs/cli/ for more information.\n`;
@@ -192,7 +192,7 @@ export class LintConfig {
       this.ignore = yaml.safeLoad(fs.readFileSync(ignoreFile, 'utf-8')) as Record<
         string,
         Record<string, Set<string>>
-      >;
+      > || {};
 
       // resolve ignore paths
       for (const fileName of Object.keys(this.ignore)) {
@@ -397,7 +397,7 @@ export class Config {
 
 export async function loadConfig(configPath?: string, customExtends?: string[]): Promise<Config> {
   if (configPath === undefined) {
-    configPath = await findConfig();
+    configPath = findConfig();
   }
 
   let rawConfig: RawConfig = {};
@@ -434,20 +434,15 @@ export async function loadConfig(configPath?: string, customExtends?: string[]):
   return new Config(rawConfig, configPath);
 }
 
-async function findConfig() {
-  if (await existsAsync('.redocly.yaml')) {
+function findConfig() {
+  if (fs.existsSync('.redocly.yaml')) {
     return '.redocly.yaml';
-  } else if (await existsAsync('.redocly.yml')) {
+  } else if (fs.existsSync('.redocly.yml')) {
     return '.redocly.yml';
   }
   return undefined;
 }
 
-function existsAsync(path: string) {
-  return new Promise(function (resolve) {
-    fs.exists(path, resolve);
-  });
-}
 
 function resolvePresets(presets: string[], plugins: Plugin[]) {
   return presets.map((presetName) => {

packages/core/src/index.ts

@@ -7,7 +7,7 @@ export { StatsAccumulator, StatsName } from './typings/common';
 export { normalizeTypes } from './types';
 export { Stats } from './rules/other/stats';
 
-export { loadConfig, Config, LintConfig } from './config/config';
+export { loadConfig, Config, LintConfig, IGNORE_FILE } from './config/config';
 export { RedoclyClient } from './redocly';
 export { BaseResolver, Document, resolveDocument, ResolveError, YamlParseError } from './resolve';
 export { unescapePointer } from './ref-utils';

packages/core/src/redocly/index.ts

@@ -136,6 +136,131 @@ export class RedoclyClient {
     );
   }
 
+  updateDefinitionVersion(definitionId: number, versionId: number, updatePatch: object): Promise<void> {
+    return this.query(`
+      mutation UpdateDefinitionVersion($definitionId: Int!, $versionId: Int!, $updatePatch: DefinitionVersionPatch!) {
+        updateDefinitionVersionByDefinitionIdAndId(input: {definitionId: $definitionId, id: $versionId, patch: $updatePatch}) {
+          definitionVersion {
+            ...VersionDetails
+            __typename
+          }
+          __typename
+        }
+      }
+      
+      fragment VersionDetails on DefinitionVersion {
+        id
+        nodeId
+        uuid
+        definitionId
+        name
+        description
+        sourceType
+        source
+        registryAccess
+        __typename
+      }
+    `,
+      {
+        definitionId,
+        versionId,
+        updatePatch,
+      },
+    );
+  }
+
+  getOrganizationId(organizationId: string) {
+    return this.query(`
+      query ($organizationId: String!) {
+        organizationById(id: $organizationId) {
+          id
+        }
+      }
+    `, {
+      organizationId
+    });
+  }
+
+  getDefinitionByName(name: string, organizationId: string) {
+    return this.query(`
+      query ($name: String!, $organizationId: String!) {
+        definition: definitionByOrganizationIdAndName(name: $name, organizationId: $organizationId) {
+          id
+        }
+      }
+    `, {
+      name,
+      organizationId
+    });
+  }
+
+  createDefinition(organizationId: string, name: string) {
+    return this.query(`
+      mutation CreateDefinition($organizationId: String!, $name: String!) {
+        def: createDefinition(input: {organizationId: $organizationId, name: $name }) {
+          definition {
+            id
+            nodeId
+            name
+          }
+        }
+      }
+    `, {
+      organizationId,
+      name
+    })
+  }
+
+  createDefinitionVersion(definitionId: string, name: string, sourceType: string, source: any) {
+    return this.query(`
+      mutation CreateVersion($definitionId: Int!, $name: String!, $sourceType: DvSourceType!, $source: JSON) {
+        createDefinitionVersion(input: {definitionId: $definitionId, name: $name, sourceType: $sourceType, source: $source }) {
+          definitionVersion {
+            id
+          }
+        }
+      }
+    `, {
+      definitionId,
+      name,
+      sourceType,
+      source
+    });
+  }
+
+  getSignedUrl(organizationId: string, filesHash: string, fileName: string) {
+    return this.query(`
+      query ($organizationId: String!, $filesHash: String!, $fileName: String!) {
+        signFileUploadCLI(organizationId: $organizationId, filesHash: $filesHash, fileName: $fileName) {
+          signedFileUrl
+          uploadedFilePath
+        }
+      }
+    `, {
+      organizationId,
+      filesHash,
+      fileName
+    })
+  }
+
+  getDefinitionVersion(organizationId: string, definitionName: string, versionName: string) {
+    return this.query(`
+      query ($organizationId: String!, $definitionName: String!, $versionName: String!) {
+        version: definitionVersionByOrganizationDefinitionAndName(organizationId: $organizationId, definitionName: $definitionName, versionName: $versionName) {
+          id
+          definitionId
+          defaultBranch {
+            name
+          }
+        }
+      }
+    `, {
+      organizationId,
+      definitionName,
+      versionName
+    });
+  }
+
   static isRegistryURL(link: string): boolean {
     const domain = process.env.REDOCLY_DOMAIN || 'redoc.ly';
     if (!link.startsWith(`https://api.${domain}/registry/`)) return false;