Complete Rewrite

Adjusted functionality to account for significantly larger files

Better organizational Structure

Added Progress Bars, added support for multiple concurrent files at once

Adjusted flow so logs are parsed and sorted as lines are read and identified, this should improve performance, reliability, and error reporting.

commands/parseLogFile.js

@@ -1,118 +1,35 @@
-import moment from "moment";
 import fs from "fs";
+import linebyline from "linebyline";
 import ora from "ora";
 import chalk from "chalk";
+import cliProgress from "cli-progress";
+import sortLog from "./sortLog.js";
 
 const spinner = ora();
-
-function importLogFile(LogFile) {
-  const allFileContents = fs.readFileSync(LogFile, "utf-8");
-  return allFileContents;
-}
-
-function getLogLines(LogFile) {
-  const fileLines = LogFile.split(/\r?\n/);
-  return fileLines;
-}
-
-function saveParsedFile(filePath, data) {
-  const outputString = JSON.stringify(data);
-  fs.writeFileSync(filePath, outputString);
-}
-
-function getDateRange(timeStamps) {
-  const sortedTimeStamps = timeStamps.sort((a, b) => b - a);
-  const dateRange = `${moment(sortedTimeStamps[1]).format(
-    "YYYY-MM-DD[T]HHmmss"
-  )}_${moment(sortedTimeStamps[sortedTimeStamps.length - 1]).format(
-    "YYYY-MM-DD[T]HHmmss"
-  )}`;
-  return dateRange;
-}
-
-function parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org) {
-  for (const logger_name of uniqueLoggers) {
-    let loggerArray = jsonArray.filter(
-      (Obj) => Obj.logger_name === logger_name
-    );
-    let loggerErrorArray = errors.filter(
-      (Obj) => Obj.logger_name === logger_name
-    );
-    let loggerFolder = `./${org}/${dateRange}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}`;
-    if (!fs.existsSync(loggerFolder)) {
-      fs.mkdirSync(loggerFolder, { recursive: true });
-    }
-    const parsedLoggerPath = `${loggerFolder}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}-${dateRange}.json`;
-    const parsedLoggerErrorsPath = `${loggerFolder}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}-${dateRange}-Errors.json`;
-
-    saveParsedFile(parsedLoggerPath, loggerArray);
-    if (loggerErrorArray.length > 0) {
-      saveParsedFile(parsedLoggerErrorsPath, loggerErrorArray);
-      console.log(chalk.yellow(`Errors found for Logger: ${logger_name}`));
-    }
-  }
-}
+// create a new progress bar instance and use shades_classic theme
+const bar1 = new cliProgress.SingleBar({
+  format: "{duration}sec | {bar} {percentage}% | {value}/{total} Bytes",
+  barCompleteChar: "\u2588",
+  barIncompleteChar: "\u2591",
+  hideCursor: true,
+  stopOnComplete: true,
+});
 
 export default function parseLogFile(LogFile) {
-  spinner.start(`Importing Log File: ${LogFile}`);
-  const wholeFile = importLogFile(LogFile);
-  spinner.succeed();
-  spinner.start(`Slicing Log File: ${LogFile}`);
-  const fileLines = getLogLines(wholeFile);
-  spinner.succeed();
-  let jsonArray = [];
-  let errors = [];
-  let loggers = [];
-  let org = null;
-  let timeStamps = [];
-
-  spinner.start(`Processing Log Lines: ${LogFile}`);
-  for (const line of fileLines) {
-    let jsonObjFromLine;
-    try {
-      jsonObjFromLine = JSON.parse(line);
-    } catch {
-      jsonObjFromLine = { text: line };
-    }
-    jsonArray.push(jsonObjFromLine);
-    if (line.includes("error")) {
-      errors.push(jsonObjFromLine);
-    }
-    if (
-      jsonObjFromLine["logger_name"] !== null &&
-      jsonObjFromLine["logger_name"] !== undefined
-    ) {
-      loggers.push(jsonObjFromLine.logger_name);
-    }
-    if (
-      jsonObjFromLine["@timestamp"] !== null &&
-      jsonObjFromLine["@timestamp"] !== undefined
-    ) {
-      timeStamps.push(new Date(jsonObjFromLine["@timestamp"]));
-    }
-    if (
-      org == null &&
-      jsonObjFromLine["org"] !== null &&
-      jsonObjFromLine["org"] !== undefined
-    ) {
-      org = jsonObjFromLine.org;
-    }
-  }
-  spinner.succeed();
-  spinner.start(`Getting Unique Loggers: ${LogFile}`);
-  const uniqueLoggers = Array.from(new Set(loggers));
-  spinner.succeed();
-  spinner.start(`Getting Date Range: ${LogFile}`);
-  const dateRange = getDateRange(timeStamps);
-  spinner.succeed();
-  parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org);
+  spinner.succeed(`Parsing Log File: ${LogFile}`);
+  let fileStats = fs.statSync(LogFile);
+  bar1.start(fileStats.size, 0, {
+    speed: "N/A",
+  });
+  let lines;
+  const rl = linebyline(LogFile);
+  rl.on("line", function (line, lineCount, byteCount) {
+    bar1.update(byteCount);
+    sortLog(line);
+    lines = lineCount;
+  });
+  rl.on("end", function (line, lineCount, byteCount) {
+    bar1.update(fileStats.size);
+    spinner.succeed(`Completed ${LogFile} / ${lines} Lines`);
+  });
 }