From baf07a79b60e40fdce6dec38d731534c6a24850f Mon Sep 17 00:00:00 2001 From: Thomas Smyth Date: Thu, 10 Dec 2020 20:46:29 +0000 Subject: [PATCH] Refactor log parser --- core/log-parser/index.js | 98 +++++++++++++++++++ .../log-parser}/log-readers/ftp.js | 2 +- .../log-parser}/log-readers/tail.js | 4 +- core/package.json | 7 +- log-parser/index.js | 87 ---------------- log-parser/package.json | 14 --- package.json | 1 - squad-server/factory.js | 13 ++- squad-server/index.js | 2 +- .../log-parser}/admin-broadcast.js | 0 .../log-parser}/index.js | 38 ++++--- .../log-parser}/new-game.js | 0 .../log-parser}/player-connected.js | 0 .../log-parser}/player-damaged.js | 0 .../log-parser}/player-died.js | 0 .../log-parser}/player-possess.js | 0 .../log-parser}/player-revived.js | 0 .../log-parser}/player-un-possess.js | 0 .../log-parser}/player-wounded.js | 0 .../log-parser}/round-winner.js | 0 .../log-parser}/server-tick-rate.js | 0 .../log-parser}/steamid-connected.js | 0 squad-server/package.json | 1 - 23 files changed, 142 insertions(+), 125 deletions(-) create mode 100644 core/log-parser/index.js rename {log-parser => core/log-parser}/log-readers/ftp.js (94%) rename {log-parser => core/log-parser}/log-readers/tail.js (96%) delete mode 100644 log-parser/index.js delete mode 100644 log-parser/package.json rename {log-parser/rules => squad-server/log-parser}/admin-broadcast.js (100%) rename {log-parser/rules => squad-server/log-parser}/index.js (56%) rename {log-parser/rules => squad-server/log-parser}/new-game.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-connected.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-damaged.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-died.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-possess.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-revived.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-un-possess.js (100%) rename {log-parser/rules => squad-server/log-parser}/player-wounded.js (100%) rename {log-parser/rules => squad-server/log-parser}/round-winner.js (100%) rename {log-parser/rules => squad-server/log-parser}/server-tick-rate.js (100%) rename {log-parser/rules => squad-server/log-parser}/steamid-connected.js (100%) diff --git a/core/log-parser/index.js b/core/log-parser/index.js new file mode 100644 index 0000000..c57f635 --- /dev/null +++ b/core/log-parser/index.js @@ -0,0 +1,98 @@ +import EventEmitter from 'events'; + +import async from 'async'; +import moment from 'moment'; + +import Logger from '../logger.js'; + +import TailLogReader from './log-readers/tail.js'; +import FTPLogReader from './log-readers/ftp.js'; + +export default class LogParser extends EventEmitter { + constructor(filename = 'filename.log', options = {}) { + super(); + + options.filename = filename; + + this.eventStrore = {}; + + this.linesPerMinute = 0; + this.matchingLinesPerMinute = 0; + this.matchingLatency = 0; + this.parsingStatsInterval = null; + + this.processLine = this.processLine.bind(this); + this.logStats = this.logStats.bind(this); + + this.queue = async.queue(this.processLine); + + switch (options.mode || 'tail') { + case 'tail': + this.logReader = new TailLogReader(this.queue.push, options); + break; + case 'ftp': + this.logReader = new FTPLogReader(this.queue.push, options); + break; + default: + throw new Error('Invalid mode.'); + } + } + + async processLine(line) { + Logger.verbose('LogParser', 4, `Matching on line: ${line}`); + + for (const rule of this.getRules()) { + const match = line.match(rule.regex); + if (!match) continue; + + Logger.verbose('LogParser', 3, `Matched on line: ${match[0]}`); + + match[1] = moment.utc(match[1], 'YYYY.MM.DD-hh.mm.ss:SSS').toDate(); + match[2] = parseInt(match[2]); + + rule.onMatch(match, this); + + this.matchingLinesPerMinute++; + this.matchingLatency += Date.now() - match[1]; + + break; + } + + this.linesPerMinute++; + } + + getRules() { + return []; + } + + async watch() { + Logger.verbose('LogParser', 1, 'Attempting to watch log file...'); + await this.logReader.watch(); + Logger.verbose('LogParser', 1, 'Watching log file...'); + + this.parsingStatsInterval = setInterval(this.logStats, 60 * 1000); + } + + logStats() { + Logger.verbose( + 'LogParser', + 1, + `Lines parsed per minute: ${ + this.linesPerMinute + } lines per minute | Matching lines per minute: ${ + this.matchingLinesPerMinute + } matching lines per minute | Average matching latency: ${ + this.matchingLatency / this.matchingLinesPerMinute + }ms` + ); + this.linesPerMinute = 0; + this.matchingLinesPerMinute = 0; + this.matchingLatency = 0; + } + + async unwatch() { + await this.logReader.unwatch(); + + clearInterval(this.parsingStatsInterval); + } +} diff --git a/log-parser/log-readers/ftp.js b/core/log-parser/log-readers/ftp.js similarity index 94% rename from log-parser/log-readers/ftp.js rename to core/log-parser/log-readers/ftp.js index 97f600d..75ecc9f 100644 --- a/log-parser/log-readers/ftp.js +++ b/core/log-parser/log-readers/ftp.js @@ -16,7 +16,7 @@ export default class TailLogReader { encoding: 'utf8', verbose: options.verbose, - path: path.join(options.logDir, 'SquadGame.log'), + path: path.join(options.logDir, options.filename), fetchInterval: options.fetchInterval || 0, maxTempFileSize: options.maxTempFileSize || 5 * 1000 * 1000, // 5 MB diff --git a/log-parser/log-readers/tail.js b/core/log-parser/log-readers/tail.js similarity index 96% rename from log-parser/log-readers/tail.js rename to core/log-parser/log-readers/tail.js index bf62c45..d50a33e 100644 --- a/log-parser/log-readers/tail.js +++ b/core/log-parser/log-readers/tail.js @@ -1,16 +1,18 @@ import path from 'path'; + import TailModule from 'tail'; export default class TailLogReader { constructor(queueLine, options = {}) { if (!('logDir' in options)) throw new Error(`logDir must be specified.`); - this.reader = new TailModule.Tail(path.join(options.logDir, 'SquadGame.log'), { + this.reader = new TailModule.Tail(path.join(options.logDir, options.filename), { useWatchFile: true }); if (typeof queueLine !== 'function') throw new Error('queueLine argument must be specified and be a function.'); + this.reader.on('line', queueLine); } diff --git a/core/package.json b/core/package.json index dd9df1b..88284d9 100644 --- a/core/package.json +++ b/core/package.json @@ -3,11 +3,16 @@ "version": "1.0.0", "type": "module", "exports": { + "./log-parser": "./log-parser/index.js", "./constants": "./constants.js", "./logger": "./logger.js", "./rcon": "./rcon.js" }, "dependencies": { - "chalk": "^4.1.0" + "async": "^3.2.0", + "chalk": "^4.1.0", + "ftp-tail": "^1.1.1", + "moment": "^2.29.1", + "tail": "^2.0.4" } } diff --git a/log-parser/index.js b/log-parser/index.js deleted file mode 100644 index 646eab4..0000000 --- a/log-parser/index.js +++ /dev/null @@ -1,87 +0,0 @@ -import EventEmitter from 'events'; - -import async from 'async'; -import moment from 'moment'; - -import Logger from 'core/logger'; - -import TailLogReader from './log-readers/tail.js'; -import FTPLogReader from './log-readers/ftp.js'; - -import rules from './rules/index.js'; - -export default class LogParser extends EventEmitter { - constructor(options = {}) { - super(); - - this.eventStore = {}; - - this.linesPerMinute = 0; - this.matchingLinesPerMinute = 0; - this.matchingLatency = 0; - this.parsingStatsInterval = null; - - this.queue = async.queue(async (line) => { - Logger.verbose('LogParser', 4, `Matching on line: ${line}`); - - for (const rule of rules) { - const match = line.match(rule.regex); - if (!match) continue; - - Logger.verbose('LogParser', 3, `Matched on line: ${match[0]}`); - - match[1] = moment.utc(match[1], 'YYYY.MM.DD-hh.mm.ss:SSS').toDate(); - match[2] = parseInt(match[2]); - - rule.onMatch(match, this); - - this.matchingLinesPerMinute++; - this.matchingLatency += Date.now() - match[1]; - - break; - } - - this.linesPerMinute++; - }); - - switch (options.mode || 'tail') { - case 'tail': - this.logReader = new TailLogReader(this.queue.push, options); - break; - case 'ftp': - this.logReader = new FTPLogReader(this.queue.push, options); - break; - default: - throw new Error('Invalid mode.'); - } - } - - async watch() { - Logger.verbose('LogParser', 1, 'Attempting to watch log file...'); - await this.logReader.watch(); - Logger.verbose('LogParser', 1, 'Watching log file...'); - - this.parsingStatsInterval = setInterval(() => { - Logger.verbose( - 'LogParser', - 1, - `Lines parsed per minute: ${ - this.linesPerMinute - } lines per minute | Matching lines per minute: ${ - this.matchingLinesPerMinute - } matching lines per minute | Average matching latency: ${ - this.matchingLatency / this.matchingLinesPerMinute - }ms` - ); - this.linesPerMinute = 0; - this.matchingLinesPerMinute = 0; - this.matchingLatency = 0; - }, 60 * 1000); - } - - async unwatch() { - await this.logReader.unwatch(); - - clearInterval(this.parsingStatsInterval); - } -} diff --git a/log-parser/package.json b/log-parser/package.json deleted file mode 100644 index b7168e9..0000000 --- a/log-parser/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "log-parser", - "version": "1.0.0", - "type": "module", - "exports": { - ".": "./index.js" - }, - "dependencies": { - "async": "^3.2.0", - "ftp-tail": "^1.1.1", - "moment": "^2.29.0", - "tail": "^2.0.4" - } -} diff --git a/package.json b/package.json index 8113f14..084a9b1 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "workspaces": [ "assets", "core", - "log-parser", "squad-server" ], "scripts": { diff --git a/squad-server/factory.js b/squad-server/factory.js index c8f522b..0c64f01 100644 --- a/squad-server/factory.js +++ b/squad-server/factory.js @@ -116,14 +116,19 @@ export default class SquadServerFactory { if (type === 'sequelize') { let connector; - if(typeof connectorConfig === 'string') { - connector = new Sequelize(connectorConfig, { logging: msg => Logger.verbose('Sequelize', 3, msg) }) + if (typeof connectorConfig === 'string') { + connector = new Sequelize(connectorConfig, { + logging: (msg) => Logger.verbose('Sequelize', 3, msg) + }); } else if (typeof connectorConfig === 'object') { - connector = new Sequelize({ ...connectorConfig, logging: msg => Logger.verbose('Sequelize', 3, msg) }); + connector = new Sequelize({ + ...connectorConfig, + logging: (msg) => Logger.verbose('Sequelize', 3, msg) + }); } else { throw new Error('Unknown sequelize connector config type.'); } - + await connector.authenticate(); return connector; } diff --git a/squad-server/index.js b/squad-server/index.js index 786389e..6d7d886 100644 --- a/squad-server/index.js +++ b/squad-server/index.js @@ -6,7 +6,7 @@ import Gamedig from 'gamedig'; import Logger from 'core/logger'; import { SQUADJS_API_DOMAIN } from 'core/constants'; -import LogParser from 'log-parser'; +import LogParser from './log-parser/index.js'; import Rcon from './rcon.js'; import { SQUADJS_VERSION } from './utils/constants.js'; diff --git a/log-parser/rules/admin-broadcast.js b/squad-server/log-parser/admin-broadcast.js similarity index 100% rename from log-parser/rules/admin-broadcast.js rename to squad-server/log-parser/admin-broadcast.js diff --git a/log-parser/rules/index.js b/squad-server/log-parser/index.js similarity index 56% rename from log-parser/rules/index.js rename to squad-server/log-parser/index.js index adc0898..8327702 100644 --- a/log-parser/rules/index.js +++ b/squad-server/log-parser/index.js @@ -1,3 +1,5 @@ +import LogParser from 'core/log-parser'; + import AdminBroadcast from './admin-broadcast.js'; import NewGame from './new-game.js'; import PlayerConnected from './player-connected.js'; @@ -11,17 +13,25 @@ import RoundWinner from './round-winner.js'; import ServerTickRate from './server-tick-rate.js'; import SteamIDConnected from './steamid-connected.js'; -export default [ - AdminBroadcast, - NewGame, - PlayerConnected, - PlayerDamaged, - PlayerDied, - PlayerPossess, - PlayerRevived, - PlayerUnPossess, - PlayerWounded, - RoundWinner, - ServerTickRate, - SteamIDConnected -]; +export default class SquadLogParser extends LogParser { + constructor(options) { + super('SquadGame.log', options); + } + + getRules() { + return [ + AdminBroadcast, + NewGame, + PlayerConnected, + PlayerDamaged, + PlayerDied, + PlayerPossess, + PlayerRevived, + PlayerUnPossess, + PlayerWounded, + RoundWinner, + ServerTickRate, + SteamIDConnected + ]; + } +} diff --git a/log-parser/rules/new-game.js b/squad-server/log-parser/new-game.js similarity index 100% rename from log-parser/rules/new-game.js rename to squad-server/log-parser/new-game.js diff --git a/log-parser/rules/player-connected.js b/squad-server/log-parser/player-connected.js similarity index 100% rename from log-parser/rules/player-connected.js rename to squad-server/log-parser/player-connected.js diff --git a/log-parser/rules/player-damaged.js b/squad-server/log-parser/player-damaged.js similarity index 100% rename from log-parser/rules/player-damaged.js rename to squad-server/log-parser/player-damaged.js diff --git a/log-parser/rules/player-died.js b/squad-server/log-parser/player-died.js similarity index 100% rename from log-parser/rules/player-died.js rename to squad-server/log-parser/player-died.js diff --git a/log-parser/rules/player-possess.js b/squad-server/log-parser/player-possess.js similarity index 100% rename from log-parser/rules/player-possess.js rename to squad-server/log-parser/player-possess.js diff --git a/log-parser/rules/player-revived.js b/squad-server/log-parser/player-revived.js similarity index 100% rename from log-parser/rules/player-revived.js rename to squad-server/log-parser/player-revived.js diff --git a/log-parser/rules/player-un-possess.js b/squad-server/log-parser/player-un-possess.js similarity index 100% rename from log-parser/rules/player-un-possess.js rename to squad-server/log-parser/player-un-possess.js diff --git a/log-parser/rules/player-wounded.js b/squad-server/log-parser/player-wounded.js similarity index 100% rename from log-parser/rules/player-wounded.js rename to squad-server/log-parser/player-wounded.js diff --git a/log-parser/rules/round-winner.js b/squad-server/log-parser/round-winner.js similarity index 100% rename from log-parser/rules/round-winner.js rename to squad-server/log-parser/round-winner.js diff --git a/log-parser/rules/server-tick-rate.js b/squad-server/log-parser/server-tick-rate.js similarity index 100% rename from log-parser/rules/server-tick-rate.js rename to squad-server/log-parser/server-tick-rate.js diff --git a/log-parser/rules/steamid-connected.js b/squad-server/log-parser/steamid-connected.js similarity index 100% rename from log-parser/rules/steamid-connected.js rename to squad-server/log-parser/steamid-connected.js diff --git a/squad-server/package.json b/squad-server/package.json index b42b63f..d46264d 100644 --- a/squad-server/package.json +++ b/squad-server/package.json @@ -8,7 +8,6 @@ "didyoumean": "^1.2.1", "discord.js": "^12.3.1", "gamedig": "^2.0.20", - "log-parser": "1.0.0", "mariadb": "^2.5.1", "mysql2": "^2.2.5", "pg": "^8.5.1",