From 14d9ff247d51e77640bc0f37464804eadc822dd7 Mon Sep 17 00:00:00 2001 From: Reto Brunner Date: Sat, 4 Nov 2023 17:19:45 +0100 Subject: [PATCH] sqlite: implement deleteMessages This is laying the foundation to build a cleaning task that's sort of database agnostic. All calls are done by acting on a "DeletionRequest" so interpretation of the config will go through a single point --- server/plugins/messageStorage/sqlite.ts | 38 +++++++- server/plugins/messageStorage/types.d.ts | 7 ++ test/plugins/sqlite.ts | 113 +++++++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) diff --git a/server/plugins/messageStorage/sqlite.ts b/server/plugins/messageStorage/sqlite.ts index ca7796ee..cd655b9d 100644 --- a/server/plugins/messageStorage/sqlite.ts +++ b/server/plugins/messageStorage/sqlite.ts @@ -7,7 +7,7 @@ import Config from "../../config"; import Msg, {Message} from "../../models/msg"; import Chan, {Channel} from "../../models/chan"; import Helper from "../../helper"; -import type {SearchResponse, SearchQuery, SearchableMessageStorage} from "./types"; +import type {SearchResponse, SearchQuery, SearchableMessageStorage, DeletionRequest} from "./types"; import Network from "../../models/network"; // TODO; type @@ -234,6 +234,11 @@ class SqliteMessageStorage implements SearchableMessageStorage { await this.serialize_run("VACUUM"); } + // helper method that vacuums the db, meant to be used by migration related cli commands + async vacuum() { + await this.serialize_run("VACUUM"); + } + async close() { if (!this.isEnabled) { return; @@ -481,6 +486,33 @@ class SqliteMessageStorage implements SearchableMessageStorage { }; } + async deleteMessages(req: DeletionRequest): Promise { + await this.initDone.promise; + let sql = "delete from messages where id in (select id from messages where\n"; + + // We roughly get a timestamp from N days before. + // We don't adjust for daylight savings time or other weird time jumps + const millisecondsInDay = 24 * 60 * 60 * 1000; + const deleteBefore = Date.now() - req.olderThanDays * millisecondsInDay; + sql += `time <= ${deleteBefore}\n`; + + let typeClause = ""; + + if (req.messageTypes !== null) { + typeClause = `type in (${req.messageTypes.map((type) => `'${type}'`).join(",")})\n`; + } + + if (typeClause) { + sql += `and ${typeClause}`; + } + + sql += "order by time asc\n"; + sql += `limit ${req.limit}\n`; + sql += ")"; + + return this.serialize_run(sql); + } + canProvideMessages() { return this.isEnabled; } @@ -488,13 +520,13 @@ class SqliteMessageStorage implements SearchableMessageStorage { private serialize_run(stmt: string, ...params: any[]): Promise { return new Promise((resolve, reject) => { this.database.serialize(() => { - this.database.run(stmt, params, (err) => { + this.database.run(stmt, params, function (err) { if (err) { reject(err); return; } - resolve(); + resolve(this.changes); // number of affected rows, `this` is re-bound by sqlite3 }); }); }); diff --git a/server/plugins/messageStorage/types.d.ts b/server/plugins/messageStorage/types.d.ts index cc305224..7e17ba54 100644 --- a/server/plugins/messageStorage/types.d.ts +++ b/server/plugins/messageStorage/types.d.ts @@ -4,6 +4,13 @@ import {Channel} from "../../models/channel"; import {Message} from "../../models/message"; import {Network} from "../../models/network"; import Client from "../../client"; +import type {MessageType} from "../../models/msg"; + +export type DeletionRequest = { + olderThanDays: number; + messageTypes: MessageType[] | null; // null means no restriction + limit: number; // -1 means unlimited +}; interface MessageStorage { isEnabled: boolean; diff --git a/test/plugins/sqlite.ts b/test/plugins/sqlite.ts index 400d3c9a..e2af20be 100644 --- a/test/plugins/sqlite.ts +++ b/test/plugins/sqlite.ts @@ -12,6 +12,7 @@ import MessageStorage, { rollbacks, } from "../../server/plugins/messageStorage/sqlite"; import sqlite3 from "sqlite3"; +import {DeletionRequest} from "../../server/plugins/messageStorage/types"; const orig_schema = [ // Schema version #1 @@ -127,6 +128,112 @@ describe("SQLite migrations", function () { }); }); +describe("SQLite unit tests", function () { + let store: MessageStorage; + + beforeEach(async function () { + store = new MessageStorage("testUser"); + await store._enable(":memory:"); + store.initDone.resolve(); + }); + + afterEach(async function () { + await store.close(); + }); + + it("deletes messages when asked to", async function () { + const baseDate = new Date(); + + const net = {uuid: "testnet"} as any; + const chan = {name: "#channel"} as any; + + for (let i = 0; i < 14; ++i) { + await store.index( + net, + chan, + new Msg({ + time: dateAddDays(baseDate, -i), + text: `msg ${i}`, + }) + ); + } + + const limit = 1; + const delReq: DeletionRequest = { + messageTypes: [MessageType.MESSAGE], + limit: limit, + olderThanDays: 2, + }; + + let deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(limit, "number of deleted messages doesn't match"); + + let id = 0; + let messages = await store.getMessages(net, chan, () => id++); + expect(messages.find((m) => m.text === "msg 13")).to.be.undefined; // oldest gets deleted first + + // let's test if it properly cleans now + delReq.limit = 100; + deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(11, "number of deleted messages doesn't match"); + messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.text)).to.have.ordered.members(["msg 1", "msg 0"]); + }); + + it("deletes only the types it should", async function () { + const baseDate = new Date(); + + const net = {uuid: "testnet"} as any; + const chan = {name: "#channel"} as any; + + for (let i = 0; i < 6; ++i) { + await store.index( + net, + chan, + new Msg({ + time: dateAddDays(baseDate, -i), + text: `msg ${i}`, + type: [ + MessageType.ACTION, + MessageType.AWAY, + MessageType.JOIN, + MessageType.PART, + MessageType.KICK, + MessageType.MESSAGE, + ][i], + }) + ); + } + + const delReq: DeletionRequest = { + messageTypes: [MessageType.ACTION, MessageType.JOIN, MessageType.KICK], + limit: 100, // effectively no limit + olderThanDays: 0, + }; + + let deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(3, "number of deleted messages doesn't match"); + + let id = 0; + let messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.type)).to.have.ordered.members([ + MessageType.MESSAGE, + MessageType.PART, + MessageType.AWAY, + ]); + + delReq.messageTypes = [ + MessageType.JOIN, // this is not in the remaining set, just here as a dummy + MessageType.PART, + MessageType.MESSAGE, + ]; + deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(2, "number of deleted messages doesn't match"); + messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.type)).to.have.ordered.members([MessageType.AWAY]); + }); +}); + describe("SQLite Message Storage", function () { // Increase timeout due to unpredictable I/O on CI services this.timeout(util.isRunningOnCI() ? 25000 : 5000); @@ -373,3 +480,9 @@ describe("SQLite Message Storage", function () { expect(fs.existsSync(expectedPath)).to.be.true; }); }); + +function dateAddDays(date: Date, days: number) { + const ret = new Date(date.valueOf()); + ret.setDate(date.getDate() + days); + return ret; +}