From 9dca5ace6ce33537614d48ffe1258b9d3b70fd19 Mon Sep 17 00:00:00 2001 From: luke-hagar-sp <98849695+luke-hagar-sp@users.noreply.github.com> Date: Tue, 4 Oct 2022 20:56:24 -0500 Subject: [PATCH] Complete Rewrite Adjusted functionality to account for significantly larger files Better organizational Structure Added Progress Bars, added support for multiple concurrent files at once Adjusted flow so logs are parsed and sorted as lines are read and identified, this should improve performance, reliability, and error reporting. --- commands/parseLogFile.js | 137 ++++++------------------------ commands/parseMultipleLogFiles.js | 45 ++++++++++ commands/sortLog.js | 32 +++++++ commands/startCLI.js | 14 ++- package-lock.json | 117 +++++++++++++++++++++---- package.json | 4 +- testing.js | 6 ++ 7 files changed, 218 insertions(+), 137 deletions(-) create mode 100644 commands/parseMultipleLogFiles.js create mode 100644 commands/sortLog.js create mode 100644 testing.js diff --git a/commands/parseLogFile.js b/commands/parseLogFile.js index 8ebf194..9009bd2 100644 --- a/commands/parseLogFile.js +++ b/commands/parseLogFile.js @@ -1,118 +1,35 @@ -import moment from "moment"; import fs from "fs"; +import linebyline from "linebyline"; import ora from "ora"; import chalk from "chalk"; +import cliProgress from "cli-progress"; +import sortLog from "./sortLog.js"; const spinner = ora(); - -function importLogFile(LogFile) { - const allFileContents = fs.readFileSync(LogFile, "utf-8"); - return allFileContents; -} - -function getLogLines(LogFile) { - const fileLines = LogFile.split(/\r?\n/); - return fileLines; -} - -function saveParsedFile(filePath, data) { - const outputString = JSON.stringify(data); - fs.writeFileSync(filePath, outputString); -} - -function getDateRange(timeStamps) { - const sortedTimeStamps = timeStamps.sort((a, b) => b - a); - const dateRange = `${moment(sortedTimeStamps[1]).format( - "YYYY-MM-DD[T]HHmmss" - )}_${moment(sortedTimeStamps[sortedTimeStamps.length - 1]).format( - "YYYY-MM-DD[T]HHmmss" - )}`; - return dateRange; -} - -function parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org) { - for (const logger_name of uniqueLoggers) { - let loggerArray = jsonArray.filter( - (Obj) => Obj.logger_name === logger_name - ); - let loggerErrorArray = errors.filter( - (Obj) => Obj.logger_name === logger_name - ); - let loggerFolder = `./${org}/${dateRange}/${logger_name.replaceAll( - ".", - "-" - )}`; - if (!fs.existsSync(loggerFolder)) { - fs.mkdirSync(loggerFolder, { recursive: true }); - } - const parsedLoggerPath = `${loggerFolder}/${logger_name.replaceAll( - ".", - "-" - )}-${dateRange}.json`; - const parsedLoggerErrorsPath = `${loggerFolder}/${logger_name.replaceAll( - ".", - "-" - )}-${dateRange}-Errors.json`; - - saveParsedFile(parsedLoggerPath, loggerArray); - if (loggerErrorArray.length > 0) { - saveParsedFile(parsedLoggerErrorsPath, loggerErrorArray); - console.log(chalk.yellow(`Errors found for Logger: ${logger_name}`)); - } - } -} +// create a new progress bar instance and use shades_classic theme +const bar1 = new cliProgress.SingleBar({ + format: "{duration}sec | {bar} {percentage}% | {value}/{total} Bytes", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591", + hideCursor: true, + stopOnComplete: true, +}); export default function parseLogFile(LogFile) { - spinner.start(`Importing Log File: ${LogFile}`); - const wholeFile = importLogFile(LogFile); - spinner.succeed(); - spinner.start(`Slicing Log File: ${LogFile}`); - const fileLines = getLogLines(wholeFile); - spinner.succeed(); - let jsonArray = []; - let errors = []; - let loggers = []; - let org = null; - let timeStamps = []; - - spinner.start(`Processing Log Lines: ${LogFile}`); - for (const line of fileLines) { - let jsonObjFromLine; - try { - jsonObjFromLine = JSON.parse(line); - } catch { - jsonObjFromLine = { text: line }; - } - jsonArray.push(jsonObjFromLine); - if (line.includes("error")) { - errors.push(jsonObjFromLine); - } - if ( - jsonObjFromLine["logger_name"] !== null && - jsonObjFromLine["logger_name"] !== undefined - ) { - loggers.push(jsonObjFromLine.logger_name); - } - if ( - jsonObjFromLine["@timestamp"] !== null && - jsonObjFromLine["@timestamp"] !== undefined - ) { - timeStamps.push(new Date(jsonObjFromLine["@timestamp"])); - } - if ( - org == null && - jsonObjFromLine["org"] !== null && - jsonObjFromLine["org"] !== undefined - ) { - org = jsonObjFromLine.org; - } - } - spinner.succeed(); - spinner.start(`Getting Unique Loggers: ${LogFile}`); - const uniqueLoggers = Array.from(new Set(loggers)); - spinner.succeed(); - spinner.start(`Getting Date Range: ${LogFile}`); - const dateRange = getDateRange(timeStamps); - spinner.succeed(); - parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org); + spinner.succeed(`Parsing Log File: ${LogFile}`); + let fileStats = fs.statSync(LogFile); + bar1.start(fileStats.size, 0, { + speed: "N/A", + }); + let lines; + const rl = linebyline(LogFile); + rl.on("line", function (line, lineCount, byteCount) { + bar1.update(byteCount); + sortLog(line); + lines = lineCount; + }); + rl.on("end", function (line, lineCount, byteCount) { + bar1.update(fileStats.size); + spinner.succeed(`Completed ${LogFile} / ${lines} Lines`); + }); } diff --git a/commands/parseMultipleLogFiles.js b/commands/parseMultipleLogFiles.js new file mode 100644 index 0000000..10b8bfc --- /dev/null +++ b/commands/parseMultipleLogFiles.js @@ -0,0 +1,45 @@ +import fs from "fs"; +import linebyline from "linebyline"; +import ora from "ora"; +import cliProgress from "cli-progress"; +import sortLog from "./sortLog.js"; + +const spinner = ora(); + +let files; + +// create new container +const multibar = new cliProgress.MultiBar({ + format: "{duration}sec | {bar} {percentage}% | {value}/{total} Bytes", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591", + hideCursor: true, + stopOnComplete: true, +}); + +async function startParse(file) { + const rl = linebyline(file.file); + rl.on("line", function (line, lineCount, byteCount) { + file.bar.update(byteCount); + sortLog(line); + }); + rl.on("end", function (line, lineCount, byteCount) { + file.bar.update(file.stats.size); + }); +} + +export default function parseMultipleLogFiles(logFiles) { + spinner.succeed(`Parsing Log Files: ${logFiles}`); + files = logFiles.map((file, index) => { + let stats = fs.statSync(file); + return { + file, + stats, + bar: multibar.create(stats.size, 0), + index, + }; + }); + files.forEach((file) => { + startParse(file); + }); +} diff --git a/commands/sortLog.js b/commands/sortLog.js new file mode 100644 index 0000000..9d85385 --- /dev/null +++ b/commands/sortLog.js @@ -0,0 +1,32 @@ +import fs from "fs"; + +export default async function sortLog(line) { + try { + const Obj = JSON.parse(line); + const parsedLine = { + date: new Date(Obj["@timestamp"]).toDateString(), + error: line.includes("error") || line.includes("exception"), + ...Obj, + }; + let logPath; + let folderPath; + if (parsedLine.error) { + folderPath = `./${parsedLine.org}/${ + parsedLine.date + }/Errors/${parsedLine.logger_name.replaceAll(".", "-")}`; + } else { + folderPath = `./${parsedLine.org}/${ + parsedLine.date + }/Everything/${parsedLine.logger_name.replaceAll(".", "-")}`; + } + logPath = `${folderPath}/log.json`; + if (fs.existsSync(folderPath)) { + fs.appendFileSync(logPath, `${JSON.stringify(parsedLine)}\n`); + } else { + fs.mkdirSync(folderPath, { recursive: true }); + fs.writeFileSync(logPath, `${JSON.stringify(parsedLine)}\n`); + } + } catch (err) { + // console.debug(err); + } +} diff --git a/commands/startCLI.js b/commands/startCLI.js index 6f7d881..3db4a1a 100644 --- a/commands/startCLI.js +++ b/commands/startCLI.js @@ -4,6 +4,7 @@ import ora from "ora"; import fs from "fs"; import parseLogFile from "./parseLogFile.js"; import chalk from "chalk"; +import parseMultipleLogFiles from "./parseMultipleLogFiles.js"; function checkDirectory(path) { const stats = fs.statSync(path); @@ -32,14 +33,12 @@ export default function startCLI() { message: "Select Log Files to Parse", name: "Files", choices: [ - new inquirer.Separator(chalk.yellow("------Folders------")), - ...directoryContents.filter((Obj) => checkDirectory(Obj) === true), new inquirer.Separator(chalk.green("------Files------")), ...directoryContents.filter((Obj) => checkDirectory(Obj) === false), ], validate(answer) { if (answer.length < 1) { - return "You must choose at least one File or Folder."; + return "You must choose at least one File."; } return true; @@ -47,11 +46,10 @@ export default function startCLI() { }, ]) .then((answers) => { - for (const logFile of answers.Files) { - try { - parseLogFile(logFile); - } catch {} + if (answers.Files.length > 1) { + parseMultipleLogFiles(answers.Files); + } else { + parseLogFile(answers.Files[0]); } - spinner.succeed("Finished Processing Log Files"); }); } diff --git a/package-lock.json b/package-lock.json index e5ac606..876b2ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,20 +1,20 @@ { "name": "va-log-parser", - "version": "0.0.2", + "version": "0.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "va-log-parser", - "version": "0.0.2", + "version": "0.0.3", "license": "MIT", "dependencies": { "boxen": "^7.0.0", - "chalk": "^5.0.1", + "cli-progress": "^3.11.2", "commander": "^9.4.0", "fs": "^0.0.1-security", "inquirer": "^9.1.0", - "moment": "^2.29.4", + "linebyline": "^1.3.0", "ora": "^6.1.2" }, "bin": { @@ -238,6 +238,54 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cli-progress": { + "version": "3.11.2", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz", + "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-progress/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-progress/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/cli-progress/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-progress/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/cli-spinners": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz", @@ -425,6 +473,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/linebyline": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/linebyline/-/linebyline-1.3.0.tgz", + "integrity": "sha512-3fpIYMrSU77OCf89hjXKuCx6vGwgWEu4N5DDCGqgZ1BF0HYy9V8IbQb/3+VWIU17iBQ83qQoUokH0AhPMOTi7w==" + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -453,14 +506,6 @@ "node": ">=6" } }, - "node_modules/moment": { - "version": "2.29.4", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", - "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", - "engines": { - "node": "*" - } - }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -828,6 +873,44 @@ "restore-cursor": "^4.0.0" } }, + "cli-progress": { + "version": "3.11.2", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz", + "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==", + "requires": { + "string-width": "^4.2.3" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, "cli-spinners": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz", @@ -950,6 +1033,11 @@ "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.2.0.tgz", "integrity": "sha512-wH+U77omcRzevfIG8dDhTS0V9zZyweakfD01FULl97+0EHiJTTZtJqxPSkIIo/SDPv/i07k/C9jAPY+jwLLeUQ==" }, + "linebyline": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/linebyline/-/linebyline-1.3.0.tgz", + "integrity": "sha512-3fpIYMrSU77OCf89hjXKuCx6vGwgWEu4N5DDCGqgZ1BF0HYy9V8IbQb/3+VWIU17iBQ83qQoUokH0AhPMOTi7w==" + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -969,11 +1057,6 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, - "moment": { - "version": "2.29.4", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", - "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" - }, "mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", diff --git a/package.json b/package.json index b697124..b1cf4b3 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,11 @@ "license": "MIT", "dependencies": { "boxen": "^7.0.0", - "chalk": "^5.0.1", + "cli-progress": "^3.11.2", "commander": "^9.4.0", "fs": "^0.0.1-security", "inquirer": "^9.1.0", - "moment": "^2.29.4", + "linebyline": "^1.3.0", "ora": "^6.1.2" } } diff --git a/testing.js b/testing.js new file mode 100644 index 0000000..906bddf --- /dev/null +++ b/testing.js @@ -0,0 +1,6 @@ +import fs from "fs"; + +fs.readFile("./logs/ccg_spva2_08092022.log", (err, data) => { + if (err) throw err; + console.log(data); +});