mirror of
https://github.com/thelounge/thelounge
synced 2024-11-22 03:53:08 +00:00
Add message indexing
This commit is contained in:
Pavel Djundik
parent
731b29c059
commit
941849eaa8
12 changed files with 243 additions and 27 deletions
|
|
@ -148,6 +148,19 @@ module.exports = {
|
|||
// @default null
|
||||
webirc: null,
|
||||
|
||||
//
|
||||
// Message logging
|
||||
// Logging is also controlled per user individually (logs variable)
|
||||
// Leave the array empty to disable all logging globally
|
||||
//
|
||||
// text: Text file per network/channel in user folder
|
||||
// sqlite: Messages are stored in SQLite, this allows them to be reloaded on server restart
|
||||
//
|
||||
// @type array
|
||||
// @default ["sqlite", "text"]
|
||||
//
|
||||
messageStorage: ["sqlite", "text"],
|
||||
|
||||
//
|
||||
// Log settings
|
||||
//
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@
|
|||
"semver": "5.5.0",
|
||||
"socket.io": "2.0.4",
|
||||
"spdy": "3.4.7",
|
||||
"sqlite3": "3.1.13",
|
||||
"thelounge-ldapjs-non-maintained-fork": "1.0.2",
|
||||
"ua-parser-js": "0.7.17",
|
||||
"urijs": "1.19.1",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const Network = require("./models/network");
|
|||
const ircFramework = require("irc-framework");
|
||||
const Helper = require("./helper");
|
||||
const UAParser = require("ua-parser-js");
|
||||
const MessageStorage = require("./plugins/sqlite");
|
||||
|
||||
module.exports = Client;
|
||||
|
||||
|
|
@ -78,8 +79,16 @@ function Client(manager, name, config = {}) {
|
|||
});
|
||||
|
||||
const client = this;
|
||||
|
||||
let delay = 0;
|
||||
|
||||
if (!Helper.config.public) {
|
||||
client.messageStorage = new MessageStorage();
|
||||
|
||||
if (client.config.log && Helper.config.messageStorage.includes("sqlite")) {
|
||||
client.messageStorage.enable(client.name);
|
||||
}
|
||||
}
|
||||
|
||||
(client.config.networks || []).forEach((n) => {
|
||||
setTimeout(function() {
|
||||
client.connect(n);
|
||||
|
|
@ -274,6 +283,8 @@ Client.prototype.connect = function(args) {
|
|||
network.irc.connect();
|
||||
|
||||
client.save();
|
||||
|
||||
channels.forEach((channel) => channel.loadMessages(client, network));
|
||||
};
|
||||
|
||||
Client.prototype.generateToken = function(callback) {
|
||||
|
|
|
|||
|
|
@ -63,11 +63,7 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
|
|||
return;
|
||||
}
|
||||
|
||||
this.messages.push(msg);
|
||||
|
||||
if (client.config.log === true) {
|
||||
writeUserLog.call(this, client, msg);
|
||||
}
|
||||
this.writeUserLog(client, msg);
|
||||
|
||||
if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
|
||||
const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
|
||||
|
|
@ -183,21 +179,72 @@ Chan.prototype.getFilteredClone = function(lastActiveChannel, lastMessage) {
|
|||
}, {});
|
||||
};
|
||||
|
||||
function writeUserLog(client, msg) {
|
||||
if (!msg.isLoggable()) {
|
||||
return false;
|
||||
Chan.prototype.writeUserLog = function(client, msg) {
|
||||
this.messages.push(msg);
|
||||
|
||||
// Does this user have logs disabled
|
||||
if (!client.config.log) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Are logs disabled server-wide
|
||||
if (Helper.config.messageStorage.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Is this particular message or channel loggable
|
||||
if (!msg.isLoggable() || !this.isLoggable()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Find the parent network where this channel is in
|
||||
const target = client.find(this.id);
|
||||
|
||||
if (!target) {
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: Something more pluggable
|
||||
if (Helper.config.messageStorage.includes("sqlite")) {
|
||||
client.messageStorage.index(target.network.uuid, this.name, msg);
|
||||
}
|
||||
|
||||
if (Helper.config.messageStorage.includes("text")) {
|
||||
userLog.write(
|
||||
client.name,
|
||||
target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs
|
||||
this.type === Chan.Type.LOBBY ? target.network.host : this.name,
|
||||
msg
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Chan.prototype.loadMessages = function(client, network) {
|
||||
if (!client.messageStorage || !this.isLoggable()) {
|
||||
return;
|
||||
}
|
||||
|
||||
client.messageStorage
|
||||
.getMessages(network, this)
|
||||
.then((messages) => {
|
||||
if (messages.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.messages.unshift(...messages);
|
||||
|
||||
if (!this.firstUnread) {
|
||||
this.firstUnread = messages[messages.length - 1].id;
|
||||
}
|
||||
|
||||
client.emit("more", {
|
||||
chan: this.id,
|
||||
messages: messages.slice(-100),
|
||||
});
|
||||
})
|
||||
.catch((err) => log.error(`Failed to load messages: ${err}`));
|
||||
};
|
||||
|
||||
Chan.prototype.isLoggable = function() {
|
||||
return this.type === Chan.Type.CHANNEL || this.type === Chan.Type.QUERY;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ class Msg {
|
|||
|
||||
isLoggable() {
|
||||
return this.type !== Msg.Type.MOTD &&
|
||||
this.type !== Msg.Type.ERROR &&
|
||||
this.type !== Msg.Type.BANLIST &&
|
||||
this.type !== Msg.Type.WHOIS;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,4 +54,5 @@ exports.input = function(network, chan, cmd, args) {
|
|||
shouldOpen: true,
|
||||
});
|
||||
this.save();
|
||||
newChan.loadMessages(this, network);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ module.exports = function(irc, network) {
|
|||
chan: chan.getFilteredClone(true),
|
||||
});
|
||||
|
||||
chan.loadMessages(client, network);
|
||||
|
||||
// Request channels' modes
|
||||
network.irc.raw("MODE", chan.name);
|
||||
} else if (data.nick === irc.user.nick) {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ module.exports = function(irc, network) {
|
|||
network: network.id,
|
||||
chan: chan.getFilteredClone(true),
|
||||
});
|
||||
chan.loadMessages(client, network);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ module.exports = function(irc, network) {
|
|||
network: network.id,
|
||||
chan: chan.getFilteredClone(true),
|
||||
});
|
||||
chan.loadMessages(client, network);
|
||||
}
|
||||
|
||||
let msg;
|
||||
|
|
|
|||
126
src/plugins/sqlite.js
Normal file
126
src/plugins/sqlite.js
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"use strict";
|
||||
|
||||
const path = require("path");
|
||||
const fsextra = require("fs-extra");
|
||||
const sqlite3 = require("sqlite3");
|
||||
const Helper = require("../helper");
|
||||
const Msg = require("../models/msg");
|
||||
|
||||
const currentSchemaVersion = 1;
|
||||
|
||||
const schema = [
|
||||
// Schema version #1
|
||||
"CREATE TABLE IF NOT EXISTS options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))",
|
||||
"CREATE TABLE IF NOT EXISTS messages (network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)",
|
||||
"CREATE INDEX IF NOT EXISTS network_channel ON messages (network, channel)",
|
||||
"CREATE INDEX IF NOT EXISTS time ON messages (time)",
|
||||
];
|
||||
|
||||
class MessageStorage {
|
||||
constructor() {
|
||||
this.isEnabled = false;
|
||||
}
|
||||
|
||||
enable(name) {
|
||||
const logsPath = path.join(Helper.getHomePath(), "logs");
|
||||
const sqlitePath = path.join(logsPath, `${name}.sqlite3`);
|
||||
|
||||
try {
|
||||
fsextra.ensureDirSync(logsPath);
|
||||
} catch (e) {
|
||||
log.error("Unable to create logs directory", e);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.isEnabled = true;
|
||||
|
||||
this.database = new sqlite3.cached.Database(sqlitePath);
|
||||
this.database.serialize(() => {
|
||||
schema.forEach((line) => this.database.run(line));
|
||||
|
||||
this.database.get("SELECT value FROM options WHERE name = 'schema_version'", (err, row) => {
|
||||
if (err) {
|
||||
return log.error(`Failed to retrieve schema version: ${err}`);
|
||||
}
|
||||
|
||||
// New table
|
||||
if (row === undefined) {
|
||||
this.database.serialize(() => this.database.run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", currentSchemaVersion));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const storedSchemaVersion = parseInt(row.value, 10);
|
||||
|
||||
if (storedSchemaVersion === currentSchemaVersion) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (storedSchemaVersion > currentSchemaVersion) {
|
||||
return log.error(`sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`);
|
||||
}
|
||||
|
||||
log.info(`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`);
|
||||
|
||||
this.database.serialize(() => this.database.run("UPDATE options SET value = ? WHERE name = 'schema_version'", currentSchemaVersion));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
index(network, channel, msg) {
|
||||
if (!this.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const clonedMsg = Object.keys(msg).reduce((newMsg, prop) => {
|
||||
// id is regenerated when messages are retrieved
|
||||
// previews are not stored because storage is cleared on lounge restart
|
||||
// type and time are stored in a separate column
|
||||
if (prop !== "id" && prop !== "previews" && prop !== "type" && prop !== "time") {
|
||||
newMsg[prop] = msg[prop];
|
||||
}
|
||||
|
||||
return newMsg;
|
||||
}, {});
|
||||
|
||||
this.database.serialize(() => this.database.run(
|
||||
"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)",
|
||||
network, channel.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Load messages for given channel on a given network and resolve a promise with loaded messages.
|
||||
*
|
||||
* @param Network network - Network object where the channel is
|
||||
* @param Chan channel - Channel object for which to load messages for
|
||||
*/
|
||||
getMessages(network, channel) {
|
||||
if (!this.isEnabled || Helper.config.maxHistory < 1) {
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.database.parallelize(() => this.database.all(
|
||||
"SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?",
|
||||
[network.uuid, channel.name.toLowerCase(), Helper.config.maxHistory],
|
||||
(err, rows) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve(rows.map((row) => {
|
||||
const msg = JSON.parse(row.msg);
|
||||
msg.time = row.time;
|
||||
msg.type = row.type;
|
||||
|
||||
return new Msg(msg);
|
||||
}).reverse());
|
||||
}
|
||||
));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = MessageStorage;
|
||||
1
test/fixtures/.gitignore
vendored
1
test/fixtures/.gitignore
vendored
|
|
@ -1,5 +1,6 @@
|
|||
# Files that may be generated by tests
|
||||
.thelounge/storage/
|
||||
.thelounge/logs/
|
||||
|
||||
# Fixtures contain fake packages, stored in a fake node_modules folder
|
||||
!.thelounge/packages/node_modules/
|
||||
|
|
|
|||
31
yarn.lock
31
yarn.lock
|
|
@ -70,8 +70,8 @@ ajv@^5.0.0, ajv@^5.1.0, ajv@^5.2.3, ajv@^5.3.0:
|
|||
json-schema-traverse "^0.3.0"
|
||||
|
||||
ajv@^6.0.1, ajv@^6.1.0:
|
||||
version "6.2.1"
|
||||
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.1.tgz#28a6abc493a2abe0fb4c8507acaedb43fa550671"
|
||||
version "6.2.0"
|
||||
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.0.tgz#afac295bbaa0152449e522742e4547c1ae9328d2"
|
||||
dependencies:
|
||||
fast-deep-equal "^1.0.0"
|
||||
fast-json-stable-stringify "^2.0.0"
|
||||
|
|
@ -1199,12 +1199,12 @@ caniuse-api@^1.5.2:
|
|||
lodash.uniq "^4.5.0"
|
||||
|
||||
caniuse-db@^1.0.30000529, caniuse-db@^1.0.30000634, caniuse-db@^1.0.30000639:
|
||||
version "1.0.30000813"
|
||||
resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000813.tgz#e0a1c603f8880ad787b2a35652b2733f32a5e29a"
|
||||
version "1.0.30000811"
|
||||
resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000811.tgz#19efb9238393d40078332c34485c818d641c4305"
|
||||
|
||||
caniuse-lite@^1.0.30000792, caniuse-lite@^1.0.30000809, caniuse-lite@^1.0.30000810:
|
||||
version "1.0.30000813"
|
||||
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000813.tgz#7b25e27fdfb8d133f3c932b01f77452140fcc6c9"
|
||||
version "1.0.30000811"
|
||||
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000811.tgz#0b6e40f2efccc27bd3cb52f91ee7ca4673d77d10"
|
||||
|
||||
capture-stack-trace@^1.0.0:
|
||||
version "1.0.0"
|
||||
|
|
@ -2087,8 +2087,8 @@ ee-first@1.1.1:
|
|||
resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
|
||||
|
||||
electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.30, electron-to-chromium@^1.3.33:
|
||||
version "1.3.36"
|
||||
resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.36.tgz#0eabf71a9ebea9013fb1cc35a390e068624f27e8"
|
||||
version "1.3.34"
|
||||
resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.34.tgz#d93498f40391bb0c16a603d8241b9951404157ed"
|
||||
|
||||
elliptic@^6.0.0:
|
||||
version "6.4.0"
|
||||
|
|
@ -4421,6 +4421,10 @@ nan@^2.3.0:
|
|||
version "2.9.2"
|
||||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.9.2.tgz#f564d75f5f8f36a6d9456cca7a6c4fe488ab7866"
|
||||
|
||||
nan@~2.7.0:
|
||||
version "2.7.0"
|
||||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46"
|
||||
|
||||
nanomatch@^1.2.9:
|
||||
version "1.2.9"
|
||||
resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.9.tgz#879f7150cb2dab7a471259066c104eee6e0fa7c2"
|
||||
|
|
@ -4486,7 +4490,7 @@ node-libs-browser@^2.0.0:
|
|||
util "^0.10.3"
|
||||
vm-browserify "0.0.4"
|
||||
|
||||
node-pre-gyp@^0.6.39:
|
||||
node-pre-gyp@^0.6.39, node-pre-gyp@~0.6.38:
|
||||
version "0.6.39"
|
||||
resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649"
|
||||
dependencies:
|
||||
|
|
@ -6272,6 +6276,13 @@ sprintf-js@~1.0.2:
|
|||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
|
||||
|
||||
sqlite3@3.1.13:
|
||||
version "3.1.13"
|
||||
resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9"
|
||||
dependencies:
|
||||
nan "~2.7.0"
|
||||
node-pre-gyp "~0.6.38"
|
||||
|
||||
sshpk@^1.7.0:
|
||||
version "1.13.1"
|
||||
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3"
|
||||
|
|
@ -6979,7 +6990,7 @@ utils-merge@1.0.1:
|
|||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
|
||||
|
||||
uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
|
||||
uuid@3.2.1, uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
|
||||
version "3.2.1"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue