Complete Rewrite

Adjusted functionality to account for significantly larger files Better organizational Structure Added Progress Bars, added support for multiple concurrent files at once Adjusted flow so logs are parsed and sorted as lines are read and identified, this should improve performance, reliability, and error reporting.
2025-12-06 04:21:55 +00:00 · 2022-10-04 20:56:24 -05:00
parent fa00c76754
commit 9dca5ace6c
7 changed files with 218 additions and 137 deletions
--- a/commands/parseLogFile.js
+++ b/commands/parseLogFile.js
@@ -1,118 +1,35 @@
-import moment from "moment";
 import fs from "fs";
+import linebyline from "linebyline";
 import ora from "ora";
 import chalk from "chalk";
+import cliProgress from "cli-progress";
+import sortLog from "./sortLog.js";

 const spinner = ora();
-
-function importLogFile(LogFile) {
-  const allFileContents = fs.readFileSync(LogFile, "utf-8");
-  return allFileContents;
-}
-
-function getLogLines(LogFile) {
-  const fileLines = LogFile.split(/\r?\n/);
-  return fileLines;
-}
-
-function saveParsedFile(filePath, data) {
-  const outputString = JSON.stringify(data);
-  fs.writeFileSync(filePath, outputString);
-}
-
-function getDateRange(timeStamps) {
-  const sortedTimeStamps = timeStamps.sort((a, b) => b - a);
-  const dateRange = `${moment(sortedTimeStamps[1]).format(
-    "YYYY-MM-DD[T]HHmmss"
-  )}_${moment(sortedTimeStamps[sortedTimeStamps.length - 1]).format(
-    "YYYY-MM-DD[T]HHmmss"
-  )}`;
-  return dateRange;
-}
-
-function parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org) {
-  for (const logger_name of uniqueLoggers) {
-    let loggerArray = jsonArray.filter(
-      (Obj) => Obj.logger_name === logger_name
-    );
-    let loggerErrorArray = errors.filter(
-      (Obj) => Obj.logger_name === logger_name
-    );
-    let loggerFolder = `./${org}/${dateRange}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}`;
-    if (!fs.existsSync(loggerFolder)) {
-      fs.mkdirSync(loggerFolder, { recursive: true });
-    }
-    const parsedLoggerPath = `${loggerFolder}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}-${dateRange}.json`;
-    const parsedLoggerErrorsPath = `${loggerFolder}/${logger_name.replaceAll(
-      ".",
-      "-"
-    )}-${dateRange}-Errors.json`;
-
-    saveParsedFile(parsedLoggerPath, loggerArray);
-    if (loggerErrorArray.length > 0) {
-      saveParsedFile(parsedLoggerErrorsPath, loggerErrorArray);
-      console.log(chalk.yellow(`Errors found for Logger: ${logger_name}`));
-    }
-  }
-}
+// create a new progress bar instance and use shades_classic theme
+const bar1 = new cliProgress.SingleBar({
+  format: "{duration}sec | {bar} {percentage}% | {value}/{total} Bytes",
+  barCompleteChar: "\u2588",
+  barIncompleteChar: "\u2591",
+  hideCursor: true,
+  stopOnComplete: true,
+});

 export default function parseLogFile(LogFile) {
-  spinner.start(`Importing Log File: ${LogFile}`);
-  const wholeFile = importLogFile(LogFile);
-  spinner.succeed();
-  spinner.start(`Slicing Log File: ${LogFile}`);
-  const fileLines = getLogLines(wholeFile);
-  spinner.succeed();
-  let jsonArray = [];
-  let errors = [];
-  let loggers = [];
-  let org = null;
-  let timeStamps = [];
-
-  spinner.start(`Processing Log Lines: ${LogFile}`);
-  for (const line of fileLines) {
-    let jsonObjFromLine;
-    try {
-      jsonObjFromLine = JSON.parse(line);
-    } catch {
-      jsonObjFromLine = { text: line };
-    }
-    jsonArray.push(jsonObjFromLine);
-    if (line.includes("error")) {
-      errors.push(jsonObjFromLine);
-    }
-    if (
-      jsonObjFromLine["logger_name"] !== null &&
-      jsonObjFromLine["logger_name"] !== undefined
-    ) {
-      loggers.push(jsonObjFromLine.logger_name);
-    }
-    if (
-      jsonObjFromLine["@timestamp"] !== null &&
-      jsonObjFromLine["@timestamp"] !== undefined
-    ) {
-      timeStamps.push(new Date(jsonObjFromLine["@timestamp"]));
-    }
-    if (
-      org == null &&
-      jsonObjFromLine["org"] !== null &&
-      jsonObjFromLine["org"] !== undefined
-    ) {
-      org = jsonObjFromLine.org;
-    }
-  }
-  spinner.succeed();
-  spinner.start(`Getting Unique Loggers: ${LogFile}`);
-  const uniqueLoggers = Array.from(new Set(loggers));
-  spinner.succeed();
-  spinner.start(`Getting Date Range: ${LogFile}`);
-  const dateRange = getDateRange(timeStamps);
-  spinner.succeed();
-  parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org);
+  spinner.succeed(`Parsing Log File: ${LogFile}`);
+  let fileStats = fs.statSync(LogFile);
+  bar1.start(fileStats.size, 0, {
+    speed: "N/A",
+  });
+  let lines;
+  const rl = linebyline(LogFile);
+  rl.on("line", function (line, lineCount, byteCount) {
+    bar1.update(byteCount);
+    sortLog(line);
+    lines = lineCount;
+  });
+  rl.on("end", function (line, lineCount, byteCount) {
+    bar1.update(fileStats.size);
+    spinner.succeed(`Completed ${LogFile} / ${lines} Lines`);
+  });
 }
--- a/commands/parseMultipleLogFiles.js
+++ b/commands/parseMultipleLogFiles.js
@@ -0,0 +1,45 @@
+import fs from "fs";
+import linebyline from "linebyline";
+import ora from "ora";
+import cliProgress from "cli-progress";
+import sortLog from "./sortLog.js";
+
+const spinner = ora();
+
+let files;
+
+// create new container
+const multibar = new cliProgress.MultiBar({
+  format: "{duration}sec | {bar} {percentage}% | {value}/{total} Bytes",
+  barCompleteChar: "\u2588",
+  barIncompleteChar: "\u2591",
+  hideCursor: true,
+  stopOnComplete: true,
+});
+
+async function startParse(file) {
+  const rl = linebyline(file.file);
+  rl.on("line", function (line, lineCount, byteCount) {
+    file.bar.update(byteCount);
+    sortLog(line);
+  });
+  rl.on("end", function (line, lineCount, byteCount) {
+    file.bar.update(file.stats.size);
+  });
+}
+
+export default function parseMultipleLogFiles(logFiles) {
+  spinner.succeed(`Parsing Log Files: ${logFiles}`);
+  files = logFiles.map((file, index) => {
+    let stats = fs.statSync(file);
+    return {
+      file,
+      stats,
+      bar: multibar.create(stats.size, 0),
+      index,
+    };
+  });
+  files.forEach((file) => {
+    startParse(file);
+  });
+}
--- a/commands/sortLog.js
+++ b/commands/sortLog.js
@@ -0,0 +1,32 @@
+import fs from "fs";
+
+export default async function sortLog(line) {
+  try {
+    const Obj = JSON.parse(line);
+    const parsedLine = {
+      date: new Date(Obj["@timestamp"]).toDateString(),
+      error: line.includes("error") || line.includes("exception"),
+      ...Obj,
+    };
+    let logPath;
+    let folderPath;
+    if (parsedLine.error) {
+      folderPath = `./${parsedLine.org}/${
+        parsedLine.date
+      }/Errors/${parsedLine.logger_name.replaceAll(".", "-")}`;
+    } else {
+      folderPath = `./${parsedLine.org}/${
+        parsedLine.date
+      }/Everything/${parsedLine.logger_name.replaceAll(".", "-")}`;
+    }
+    logPath = `${folderPath}/log.json`;
+    if (fs.existsSync(folderPath)) {
+      fs.appendFileSync(logPath, `${JSON.stringify(parsedLine)}\n`);
+    } else {
+      fs.mkdirSync(folderPath, { recursive: true });
+      fs.writeFileSync(logPath, `${JSON.stringify(parsedLine)}\n`);
+    }
+  } catch (err) {
+    // console.debug(err);
+  }
+}
--- a/commands/startCLI.js
+++ b/commands/startCLI.js
@@ -4,6 +4,7 @@ import ora from "ora";
 import fs from "fs";
 import parseLogFile from "./parseLogFile.js";
 import chalk from "chalk";
+import parseMultipleLogFiles from "./parseMultipleLogFiles.js";

 function checkDirectory(path) {
  const stats = fs.statSync(path);
@@ -32,14 +33,12 @@ export default function startCLI() {
        message: "Select Log Files to Parse",
        name: "Files",
        choices: [
-          new inquirer.Separator(chalk.yellow("------Folders------")),
-          ...directoryContents.filter((Obj) => checkDirectory(Obj) === true),
          new inquirer.Separator(chalk.green("------Files------")),
          ...directoryContents.filter((Obj) => checkDirectory(Obj) === false),
        ],
        validate(answer) {
          if (answer.length < 1) {
-            return "You must choose at least one File or Folder.";
+            return "You must choose at least one File.";
          }

          return true;
@@ -47,11 +46,10 @@ export default function startCLI() {
      },
    ])
    .then((answers) => {
-      for (const logFile of answers.Files) {
-        try {
-          parseLogFile(logFile);
-        } catch {}
+      if (answers.Files.length > 1) {
+        parseMultipleLogFiles(answers.Files);
+      } else {
+        parseLogFile(answers.Files[0]);
      }
-      spinner.succeed("Finished Processing Log Files");
    });
 }
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,20 +1,20 @@
 {
  "name": "va-log-parser",
-  "version": "0.0.2",
+  "version": "0.0.3",
  "lockfileVersion": 2,
  "requires": true,
  "packages": {
    "": {
      "name": "va-log-parser",
-      "version": "0.0.2",
+      "version": "0.0.3",
      "license": "MIT",
      "dependencies": {
        "boxen": "^7.0.0",
-        "chalk": "^5.0.1",
+        "cli-progress": "^3.11.2",
        "commander": "^9.4.0",
        "fs": "^0.0.1-security",
        "inquirer": "^9.1.0",
-        "moment": "^2.29.4",
+        "linebyline": "^1.3.0",
        "ora": "^6.1.2"
      },
      "bin": {
@@ -238,6 +238,54 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/cli-progress": {
+      "version": "3.11.2",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
+      "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
+      "dependencies": {
+        "string-width": "^4.2.3"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/cli-progress/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/cli-progress/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/cli-progress/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/cli-progress/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/cli-spinners": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz",
@@ -425,6 +473,11 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/linebyline": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/linebyline/-/linebyline-1.3.0.tgz",
+      "integrity": "sha512-3fpIYMrSU77OCf89hjXKuCx6vGwgWEu4N5DDCGqgZ1BF0HYy9V8IbQb/3+VWIU17iBQ83qQoUokH0AhPMOTi7w=="
+    },
    "node_modules/lodash": {
      "version": "4.17.21",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -453,14 +506,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/moment": {
-      "version": "2.29.4",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
-      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
-      "engines": {
-        "node": "*"
-      }
-    },
    "node_modules/mute-stream": {
      "version": "0.0.8",
      "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
@@ -828,6 +873,44 @@
        "restore-cursor": "^4.0.0"
      }
    },
+    "cli-progress": {
+      "version": "3.11.2",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
+      "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
+      "requires": {
+        "string-width": "^4.2.3"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+          "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
+        },
+        "emoji-regex": {
+          "version": "8.0.0",
+          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+          "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+        },
+        "string-width": {
+          "version": "4.2.3",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+          "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.1"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+          "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+          "requires": {
+            "ansi-regex": "^5.0.1"
+          }
+        }
+      }
+    },
    "cli-spinners": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz",
@@ -950,6 +1033,11 @@
      "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.2.0.tgz",
      "integrity": "sha512-wH+U77omcRzevfIG8dDhTS0V9zZyweakfD01FULl97+0EHiJTTZtJqxPSkIIo/SDPv/i07k/C9jAPY+jwLLeUQ=="
    },
+    "linebyline": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/linebyline/-/linebyline-1.3.0.tgz",
+      "integrity": "sha512-3fpIYMrSU77OCf89hjXKuCx6vGwgWEu4N5DDCGqgZ1BF0HYy9V8IbQb/3+VWIU17iBQ83qQoUokH0AhPMOTi7w=="
+    },
    "lodash": {
      "version": "4.17.21",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -969,11 +1057,6 @@
      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
      "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
    },
-    "moment": {
-      "version": "2.29.4",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
-      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
-    },
    "mute-stream": {
      "version": "0.0.8",
      "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
--- a/package.json
+++ b/package.json
@@ -21,11 +21,11 @@
  "license": "MIT",
  "dependencies": {
    "boxen": "^7.0.0",
-    "chalk": "^5.0.1",
+    "cli-progress": "^3.11.2",
    "commander": "^9.4.0",
    "fs": "^0.0.1-security",
    "inquirer": "^9.1.0",
-    "moment": "^2.29.4",
+    "linebyline": "^1.3.0",
    "ora": "^6.1.2"
  }
 }
--- a/testing.js
+++ b/testing.js
@@ -0,0 +1,6 @@
+import fs from "fs";
+
+fs.readFile("./logs/ccg_spva2_08092022.log", (err, data) => {
+  if (err) throw err;
+  console.log(data);
+});