VA-Log-Parser/commands/parseLogFile.js

import moment from "moment";
import fs from "fs";
import ora from "ora";
import chalk from "chalk";

const spinner = ora();

function importLogFile(LogFile) {
  const allFileContents = fs.readFileSync(LogFile, "utf-8");
  return allFileContents;
}

function getLogLines(LogFile) {
  const fileLines = LogFile.split(/\r?\n/);
  return fileLines;
}

function checkDirectory(path) {
  const stats = fs.statSync(path);
  return stats.isDirectory();
}

function saveParsedFile(filePath, data) {
  const outputString = JSON.stringify(data);
  fs.writeFileSync(filePath, outputString);
}

function getDateRange(timeStamps) {
  const sortedTimeStamps = timeStamps.sort((a, b) => b - a);
  const dateRange = `${moment(sortedTimeStamps[1]).format(
    "YYYY-MM-DD[T]HHmmss"
  )}_${moment(sortedTimeStamps[sortedTimeStamps.length - 1]).format(
    "YYYY-MM-DD[T]HHmmss"
  )}`;
  return dateRange;
}

function parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org) {
  for (const logger_name of uniqueLoggers) {
    let loggerArray = jsonArray.filter(
      (Obj) => Obj.logger_name === logger_name
    );
    let loggerErrorArray = errors.filter(
      (Obj) => Obj.logger_name === logger_name
    );
    let loggerFolder = `./${org}/${dateRange}/${logger_name.replaceAll(
      ".",
      "-"
    )}`;
    if (!fs.existsSync(loggerFolder)) {
      fs.mkdirSync(loggerFolder, { recursive: true });
    }
    const parsedLoggerPath = `${loggerFolder}/${logger_name.replaceAll(
      ".",
      "-"
    )}-${dateRange}.json`;
    const parsedLoggerErrorsPath = `${loggerFolder}/${logger_name.replaceAll(
      ".",
      "-"
    )}-${dateRange}-Errors.json`;

    saveParsedFile(parsedLoggerPath, loggerArray);
    if (loggerErrorArray.length > 0) {
      saveParsedFile(parsedLoggerErrorsPath, loggerErrorArray);
      console.log(chalk.yellow(`Errors found for Logger: ${logger_name}`));
    }
  }
}

export default function parseLogFile(LogFile) {
  spinner.start(`Importing Log File: ${LogFile}`);
  const wholeFile = importLogFile(LogFile);
  spinner.succeed();
  spinner.start(`Slicing Log File: ${LogFile}`);
  const fileLines = getLogLines(wholeFile);
  spinner.succeed();
  let jsonArray = [];
  let errors = [];
  let loggers = [];
  let org = null;
  let timeStamps = [];

  spinner.start(`Processing Log Lines: ${LogFile}`);
  for (const line of fileLines) {
    let jsonObjFromLine;
    try {
      jsonObjFromLine = JSON.parse(line);
    } catch {
      jsonObjFromLine = { text: line };
    }
    jsonArray.push(jsonObjFromLine);
    if (line.includes("error")) {
      errors.push(jsonObjFromLine);
    }
    if (
      jsonObjFromLine["logger_name"] !== null &&
      jsonObjFromLine["logger_name"] !== undefined
    ) {
      loggers.push(jsonObjFromLine.logger_name);
    }
    if (
      jsonObjFromLine["@timestamp"] !== null &&
      jsonObjFromLine["@timestamp"] !== undefined
    ) {
      timeStamps.push(new Date(jsonObjFromLine["@timestamp"]));
    }
    if (
      org == null &&
      jsonObjFromLine["org"] !== null &&
      jsonObjFromLine["org"] !== undefined
    ) {
      org = jsonObjFromLine.org;
    }
  }
  spinner.succeed();
  spinner.start(`Getting Unique Loggers: ${LogFile}`);
  const uniqueLoggers = Array.from(new Set(loggers));
  spinner.succeed();
  spinner.start(`Getting Date Range: ${LogFile}`);
  const dateRange = getDateRange(timeStamps);
  spinner.succeed();
  parseLoggers(uniqueLoggers, jsonArray, errors, dateRange, org);
}