Mirrors/thelounge
2
0
Fork 0
mirror of https://github.com/thelounge/thelounge synced 2024-11-25 13:30:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

Store preview images on disk for privacy, security and caching

Browse source
This commit is contained in:
Pavel Djundik 2017-07-06 18:33:09 +03:00
parent ce0e460368
commit f35a2809a7
10 changed files with 244 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -4,6 +4,7 @@ package-lock.json
.nyc_output/
coverage/
test/fixtures/.lounge/storage/
# Built assets created at npm install/prepublish time
# See https://docs.npmjs.com/misc/scripts

2
client/views/msg_preview.tpl
View file

@ -2,7 +2,7 @@
<div class="toggle-content toggle-type-{{type}}{{#if shown}} show{{/if}}">
{{#equal type "image"}}
<a class="toggle-thumbnail" href="{{link}}" target="_blank" rel="noopener">
<img src="{{link}}">
<img src="{{thumb}}">
</a>
{{else}}
{{#if thumb}}

17
defaults/config.js
View file

@ -66,6 +66,23 @@ module.exports = {
//
prefetch: false,
//
// Store and proxy prefetched images and thumbnails.
// This improves security and privacy by not exposing client IP address,
// and always loading images from The Lounge instance and making all assets secure,
// which in result fixes mixed content warnings.
//
// If storage is enabled, The Lounge will fetch and store images and thumbnails
// in ~/.lounge/storage folder, or %HOME%/storage if --home is used.
//
// Images are deleted when they are no longer referenced by any message (controlled by maxHistory),
// and the folder is cleaned up on every The Lounge restart.
//
// @type boolean
// @default false
//
prefetchStorage: false,
//
// Prefetch URLs Image Preview size limit
//

5
src/helper.js
View file

@ -12,6 +12,7 @@ const colors = require("colors/safe");
var Helper = {
config: null,
expandHome: expandHome,
getStoragePath: getStoragePath,
getUserConfigPath: getUserConfigPath,
getUserLogsPath: getUserLogsPath,
setHome: setHome,
@ -90,6 +91,10 @@ function getUserLogsPath(name, network) {
return path.join(this.HOME, "logs", name, network);
}
function getStoragePath() {
return path.join(this.HOME, "storage");
}
function ip2hex(address) {
// no ipv6 support
if (!net.isIPv4(address)) {

11
src/models/chan.js
View file

@ -2,6 +2,7 @@
var _ = require("lodash");
var Helper = require("../helper");
const storage = require("../plugins/storage");
module.exports = Chan;
@ -53,7 +54,15 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
this.messages.push(msg);
if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
if (Helper.config.prefetch && Helper.config.prefetchStorage) {
deleted.forEach((deletedMessage) => {
if (deletedMessage.preview && deletedMessage.preview.thumb) {
storage.dereference(deletedMessage.preview.thumb);
}
});
}
}
if (!msg.self && !isOpen) {

37
src/plugins/irc-events/link.js
View file

@ -5,6 +5,7 @@ const request = require("request");
const Helper = require("../../helper");
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
const es = require("event-stream");
const storage = require("../storage");
process.setMaxListeners(0);
@ -49,7 +50,7 @@ function parse(msg, url, res, client) {
switch (res.type) {
case "text/html":
var $ = cheerio.load(res.text);
var $ = cheerio.load(res.data);
preview.type = "link";
preview.head =
$("meta[property=\"og:title\"]").attr("content")
@ -78,7 +79,7 @@ function parse(msg, url, res, client) {
preview.thumb = "";
}
emitPreview(client, msg, preview);
handlePreview(client, msg, preview, resThumb);
});
return;
@ -90,18 +91,32 @@ function parse(msg, url, res, client) {
case "image/gif":
case "image/jpg":
case "image/jpeg":
if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) {
preview.type = "image";
} else {
if (res.size > (Helper.config.prefetchMaxImageSize * 1024)) {
return;
}
preview.type = "image";
preview.thumb = preview.link;
break;
default:
return;
}
emitPreview(client, msg, preview);
handlePreview(client, msg, preview, res);
}
function handlePreview(client, msg, preview, res) {
if (!preview.thumb.length || !Helper.config.prefetchStorage) {
return emitPreview(client, msg, preview);
}
storage.store(res.data, res.type.replace("image/", ""), (url) => {
preview.thumb = url;
emitPreview(client, msg, preview);
});
}
function emitPreview(client, msg, preview) {
@ -164,23 +179,23 @@ function fetch(url, cb) {
return cb(null);
}
let type;
let type = "";
let size = parseInt(req.response.headers["content-length"], 10) || length;
if (size < length) {
size = length;
}
try {
if (req.response.headers["content-type"]) {
type = req.response.headers["content-type"].split(/ *; */).shift();
} catch (e) {
type = {};
}
data = {
text: data,
data: data,
type: type,
size: size
};
cb(data);
}));
}

81
src/plugins/storage.js Normal file
View file

@ -0,0 +1,81 @@
"use strict";
const fs = require("fs");
const fsextra = require("fs-extra");
const path = require("path");
const crypto = require("crypto");
const helper = require("../helper");
class Storage {
constructor() {
this.references = new Map();
// Ensures that a directory is empty.
// Deletes directory contents if the directory is not empty.
// If the directory does not exist, it is created.
fsextra.emptyDirSync(helper.getStoragePath());
}
dereference(url) {
// If maxHistory is 0, image would be dereferenced before client had a chance to retrieve it,
// so for now, just don't implement dereferencing for this edge case.
if (helper.maxHistory === 0) {
return;
}
const references = (this.references.get(url) || 0) - 1;
if (references < 0) {
return log.warn("Tried to dereference a file that has no references", url);
}
if (references > 0) {
return this.references.set(url, references);
}
this.references.delete(url);
// Drop "storage/" from url and join it with full storage path
const filePath = path.join(helper.getStoragePath(), url.substring(8));
fs.unlink(filePath, (err) => {
if (err) {
log.error("Failed to delete stored file", err);
}
});
}
store(data, extension, callback) {
const hash = crypto.createHash("sha256").update(data).digest("hex");
const a = hash.substring(0, 2);
const b = hash.substring(2, 4);
const folder = path.join(helper.getStoragePath(), a, b);
const filePath = path.join(folder, `${hash.substring(4)}.${extension}`);
const url = `storage/${a}/${b}/${hash.substring(4)}.${extension}`;
this.references.set(url, 1 + (this.references.get(url) || 0));
// If file with this name already exists, we don't need to write it again
if (fs.existsSync(filePath)) {
return callback(url);
}
fsextra.ensureDir(folder).then(() => {
fs.writeFile(filePath, data, (err) => {
if (err) {
log.error("Failed to store a file", err);
return callback("");
}
callback(url);
});
}).catch((err) => {
log.error("Failed to create storage folder", err);
return callback("");
});
}
}
module.exports = new Storage();

23
src/server.js
View file

@ -33,6 +33,10 @@ module.exports = function() {
.use(allRequests)
.use(index)
.use(express.static("client"))
.use("/storage/", express.static(Helper.getStoragePath(), {
redirect: false,
maxAge: 86400 * 1000,
}))
.engine("html", expressHandlebars({
extname: ".html",
helpers: {
@ -152,7 +156,24 @@ function index(req, res, next) {
filename: filename
};
});
res.setHeader("Content-Security-Policy", "default-src *; connect-src 'self' ws: wss:; style-src * 'unsafe-inline'; script-src 'self'; child-src 'self'; object-src 'none'; form-action 'none';");
const policies = [
"default-src *",
"connect-src 'self' ws: wss:",
"style-src * 'unsafe-inline'",
"script-src 'self'",
"child-src 'self'",
"object-src 'none'",
"form-action 'none'",
];
// If prefetch is enabled, but storage is not, we have to allow mixed content
if (Helper.config.prefetchStorage || !Helper.config.prefetch) {
policies.push("img-src 'self'");
policies.unshift("block-all-mixed-content");
}
res.setHeader("Content-Security-Policy", policies.join("; "));
res.setHeader("Referrer-Policy", "no-referrer");
res.render("index", data);
}

18
test/plugins/link.js
View file

@ -1,10 +1,10 @@
"use strict";
const expect = require("chai").expect;
var util = require("../util");
var link = require("../../src/plugins/irc-events/link.js");
const path = require("path");
const expect = require("chai").expect;
const util = require("../util");
const Helper = require("../../src/helper");
const link = require("../../src/plugins/irc-events/link.js");
describe("Link plugin", function() {
before(function(done) {
@ -22,6 +22,8 @@ describe("Link plugin", function() {
beforeEach(function() {
this.irc = util.createClient();
this.network = util.createNetwork();
Helper.config.prefetchStorage = false;
});
it("should be able to fetch basic information about URLs", function(done) {
@ -39,6 +41,7 @@ describe("Link plugin", function() {
expect(data.preview.type).to.equal("link");
expect(data.preview.head).to.equal("test title");
expect(data.preview.body).to.equal("simple description");
expect(data.preview.link).to.equal("http://localhost:9002/basic");
expect(message.previews.length).to.equal(1);
done();
});
@ -104,11 +107,13 @@ describe("Link plugin", function() {
link(this.irc, this.network.channels[0], message);
this.app.get("/invalid-thumb", function(req, res) {
res.send("<title>test</title><meta property='og:image' content='/real-test-image.png'>");
res.send("<title>test invalid image</title><meta property='og:image' content='/real-test-image.png'>");
});
this.irc.once("msg:preview", function(data) {
expect(data.preview.thumb).to.be.empty;
expect(data.preview.head).to.equal("test invalid image");
expect(data.preview.link).to.equal("http://localhost:9002/invalid-thumb");
done();
});
});
@ -127,6 +132,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.head).to.equal("Untitled page");
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
expect(data.preview.link).to.equal("http://localhost:9002/thumb-no-title");
done();
});
});
@ -144,6 +150,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.head).to.equal("404 image");
expect(data.preview.link).to.equal("http://localhost:9002/thumb-404");
expect(data.preview.thumb).to.be.empty;
done();
});
@ -159,6 +166,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.type).to.equal("image");
expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png");
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
done();
});
});

68
test/plugins/storage.js Normal file
View file

@ -0,0 +1,68 @@
"use strict";
const fs = require("fs");
const path = require("path");
const crypto = require("crypto");
const expect = require("chai").expect;
const util = require("../util");
const Helper = require("../../src/helper");
const link = require("../../src/plugins/irc-events/link.js");
describe("Image storage", function() {
const testImagePath = path.resolve(__dirname, "../../client/img/apple-touch-icon-120x120.png");
const correctImageHash = crypto.createHash("sha256").update(fs.readFileSync(testImagePath)).digest("hex");
const correctImageURL = `storage/${correctImageHash.substring(0, 2)}/${correctImageHash.substring(2, 4)}/${correctImageHash.substring(4)}.png`;
before(function(done) {
this.app = util.createWebserver();
this.app.get("/real-test-image.png", function(req, res) {
res.sendFile(testImagePath);
});
this.connection = this.app.listen(9003, done);
});
after(function(done) {
this.connection.close(done);
});
beforeEach(function() {
this.irc = util.createClient();
this.network = util.createNetwork();
Helper.config.prefetchStorage = true;
});
it("should store the thumbnail", function(done) {
const message = this.irc.createMessage({
text: "http://localhost:9003/thumb"
});
link(this.irc, this.network.channels[0], message);
this.app.get("/thumb", function(req, res) {
res.send("<title>Google</title><meta property='og:image' content='http://localhost:9003/real-test-image.png'>");
});
this.irc.once("msg:preview", function(data) {
expect(data.preview.head).to.equal("Google");
expect(data.preview.link).to.equal("http://localhost:9003/thumb");
expect(data.preview.thumb).to.equal(correctImageURL);
done();
});
});
it("should store the image", function(done) {
const message = this.irc.createMessage({
text: "http://localhost:9003/real-test-image.png"
});
link(this.irc, this.network.channels[0], message);
this.irc.once("msg:preview", function(data) {
expect(data.preview.type).to.equal("image");
expect(data.preview.link).to.equal("http://localhost:9003/real-test-image.png");
expect(data.preview.thumb).to.equal(correctImageURL);
done();
});
});
});