sqlite: implement deleteMessages

This is laying the foundation to build a cleaning task that's
sort of database agnostic.
All calls are done by acting on a "DeletionRequest" so interpretation
of the config will go through a single point

server/plugins/messageStorage/sqlite.ts

@@ -7,7 +7,7 @@ import Config from "../../config";
 import Msg, {Message} from "../../models/msg";
 import Chan, {Channel} from "../../models/chan";
 import Helper from "../../helper";
-import type {SearchResponse, SearchQuery, SearchableMessageStorage} from "./types";
+import type {SearchResponse, SearchQuery, SearchableMessageStorage, DeletionRequest} from "./types";
 import Network from "../../models/network";
 
 // TODO; type
@@ -234,6 +234,11 @@ class SqliteMessageStorage implements SearchableMessageStorage {
 		await this.serialize_run("VACUUM");
 	}
 
+	// helper method that vacuums the db, meant to be used by migration related cli commands
+	async vacuum() {
+		await this.serialize_run("VACUUM");
+	}
+
 	async close() {
 		if (!this.isEnabled) {
 			return;
@@ -481,6 +486,33 @@ class SqliteMessageStorage implements SearchableMessageStorage {
 		};
 	}
 
+	async deleteMessages(req: DeletionRequest): Promise<number> {
+		await this.initDone.promise;
+		let sql = "delete from messages where id in (select id from messages where\n";
+
+		// We roughly get a timestamp from N days before.
+		// We don't adjust for daylight savings time or other weird time jumps
+		const millisecondsInDay = 24 * 60 * 60 * 1000;
+		const deleteBefore = Date.now() - req.olderThanDays * millisecondsInDay;
+		sql += `time <= ${deleteBefore}\n`;
+
+		let typeClause = "";
+
+		if (req.messageTypes !== null) {
+			typeClause = `type in (${req.messageTypes.map((type) => `'${type}'`).join(",")})\n`;
+		}
+
+		if (typeClause) {
+			sql += `and ${typeClause}`;
+		}
+
+		sql += "order by time asc\n";
+		sql += `limit ${req.limit}\n`;
+		sql += ")";
+
+		return this.serialize_run(sql);
+	}
+
 	canProvideMessages() {
 		return this.isEnabled;
 	}
@@ -488,13 +520,13 @@ class SqliteMessageStorage implements SearchableMessageStorage {
 	private serialize_run(stmt: string, ...params: any[]): Promise<number> {
 		return new Promise((resolve, reject) => {
 			this.database.serialize(() => {
-				this.database.run(stmt, params, (err) => {
+				this.database.run(stmt, params, function (err) {
 					if (err) {
 						reject(err);
 						return;
 					}
 
-					resolve();
+					resolve(this.changes); // number of affected rows, `this` is re-bound by sqlite3
 				});
 			});
 		});

server/plugins/messageStorage/types.d.ts

@@ -4,6 +4,13 @@ import {Channel} from "../../models/channel";
 import {Message} from "../../models/message";
 import {Network} from "../../models/network";
 import Client from "../../client";
+import type {MessageType} from "../../models/msg";
+
+export type DeletionRequest = {
+	olderThanDays: number;
+	messageTypes: MessageType[] | null; // null means no restriction
+	limit: number; // -1 means unlimited
+};
 
 interface MessageStorage {
 	isEnabled: boolean;

test/plugins/sqlite.ts

@@ -12,6 +12,7 @@ import MessageStorage, {
 	rollbacks,
 } from "../../server/plugins/messageStorage/sqlite";
 import sqlite3 from "sqlite3";
+import {DeletionRequest} from "../../server/plugins/messageStorage/types";
 
 const orig_schema = [
 	// Schema version #1
@@ -127,6 +128,112 @@ describe("SQLite migrations", function () {
 	});
 });
 
+describe("SQLite unit tests", function () {
+	let store: MessageStorage;
+
+	beforeEach(async function () {
+		store = new MessageStorage("testUser");
+		await store._enable(":memory:");
+		store.initDone.resolve();
+	});
+
+	afterEach(async function () {
+		await store.close();
+	});
+
+	it("deletes messages when asked to", async function () {
+		const baseDate = new Date();
+
+		const net = {uuid: "testnet"} as any;
+		const chan = {name: "#channel"} as any;
+
+		for (let i = 0; i < 14; ++i) {
+			await store.index(
+				net,
+				chan,
+				new Msg({
+					time: dateAddDays(baseDate, -i),
+					text: `msg ${i}`,
+				})
+			);
+		}
+
+		const limit = 1;
+		const delReq: DeletionRequest = {
+			messageTypes: [MessageType.MESSAGE],
+			limit: limit,
+			olderThanDays: 2,
+		};
+
+		let deleted = await store.deleteMessages(delReq);
+		expect(deleted).to.equal(limit, "number of deleted messages doesn't match");
+
+		let id = 0;
+		let messages = await store.getMessages(net, chan, () => id++);
+		expect(messages.find((m) => m.text === "msg 13")).to.be.undefined; // oldest gets deleted first
+
+		// let's test if it properly cleans now
+		delReq.limit = 100;
+		deleted = await store.deleteMessages(delReq);
+		expect(deleted).to.equal(11, "number of deleted messages doesn't match");
+		messages = await store.getMessages(net, chan, () => id++);
+		expect(messages.map((m) => m.text)).to.have.ordered.members(["msg 1", "msg 0"]);
+	});
+
+	it("deletes only the types it should", async function () {
+		const baseDate = new Date();
+
+		const net = {uuid: "testnet"} as any;
+		const chan = {name: "#channel"} as any;
+
+		for (let i = 0; i < 6; ++i) {
+			await store.index(
+				net,
+				chan,
+				new Msg({
+					time: dateAddDays(baseDate, -i),
+					text: `msg ${i}`,
+					type: [
+						MessageType.ACTION,
+						MessageType.AWAY,
+						MessageType.JOIN,
+						MessageType.PART,
+						MessageType.KICK,
+						MessageType.MESSAGE,
+					][i],
+				})
+			);
+		}
+
+		const delReq: DeletionRequest = {
+			messageTypes: [MessageType.ACTION, MessageType.JOIN, MessageType.KICK],
+			limit: 100, // effectively no limit
+			olderThanDays: 0,
+		};
+
+		let deleted = await store.deleteMessages(delReq);
+		expect(deleted).to.equal(3, "number of deleted messages doesn't match");
+
+		let id = 0;
+		let messages = await store.getMessages(net, chan, () => id++);
+		expect(messages.map((m) => m.type)).to.have.ordered.members([
+			MessageType.MESSAGE,
+			MessageType.PART,
+			MessageType.AWAY,
+		]);
+
+		delReq.messageTypes = [
+			MessageType.JOIN, // this is not in the remaining set, just here as a dummy
+			MessageType.PART,
+			MessageType.MESSAGE,
+		];
+		deleted = await store.deleteMessages(delReq);
+		expect(deleted).to.equal(2, "number of deleted messages doesn't match");
+		messages = await store.getMessages(net, chan, () => id++);
+		expect(messages.map((m) => m.type)).to.have.ordered.members([MessageType.AWAY]);
+	});
+});
+
 describe("SQLite Message Storage", function () {
 	// Increase timeout due to unpredictable I/O on CI services
 	this.timeout(util.isRunningOnCI() ? 25000 : 5000);
@@ -373,3 +480,9 @@ describe("SQLite Message Storage", function () {
 		expect(fs.existsSync(expectedPath)).to.be.true;
 	});
 });
+
+function dateAddDays(date: Date, days: number) {
+	const ret = new Date(date.valueOf());
+	ret.setDate(date.getDate() + days);
+	return ret;
+}