Merge branch 'main' into plugin-system

This commit is contained in:
Felix Ableitner 2024-05-15 11:51:32 +02:00
commit 2ef6e2e43b
27 changed files with 1477 additions and 2284 deletions

30
Cargo.lock generated
View file

@ -16,9 +16,9 @@ checksum = "8f27d075294830fcab6f66e320dab524bc6d048f4a151698e153205559113772"
[[package]]
name = "activitypub_federation"
version = "0.5.5"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e54fe65c4a4b57712d8e436f1fb86ff37e10b56f011f4233fbbfa8c669163e9d"
checksum = "ac8ff2d0151ce9ac02eb29e4a58b41d28693f141f7963d4bfabd2f9d402ecec7"
dependencies = [
"activitystreams-kinds",
"actix-web",
@ -3106,7 +3106,7 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]]
name = "lemmy_api"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"actix-web",
@ -3135,7 +3135,7 @@ dependencies = [
[[package]]
name = "lemmy_api_common"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"actix-web",
@ -3173,7 +3173,7 @@ dependencies = [
[[package]]
name = "lemmy_api_crud"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"accept-language",
"activitypub_federation",
@ -3196,7 +3196,7 @@ dependencies = [
[[package]]
name = "lemmy_apub"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"actix-web",
@ -3236,7 +3236,7 @@ dependencies = [
[[package]]
name = "lemmy_db_perf"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"anyhow",
"clap",
@ -3251,7 +3251,7 @@ dependencies = [
[[package]]
name = "lemmy_db_schema"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"anyhow",
@ -3291,7 +3291,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"actix-web",
"chrono",
@ -3313,7 +3313,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views_actor"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"chrono",
"diesel",
@ -3333,7 +3333,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views_moderator"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"diesel",
"diesel-async",
@ -3345,7 +3345,7 @@ dependencies = [
[[package]]
name = "lemmy_federate"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"anyhow",
@ -3368,7 +3368,7 @@ dependencies = [
[[package]]
name = "lemmy_routes"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"actix-web",
@ -3393,7 +3393,7 @@ dependencies = [
[[package]]
name = "lemmy_server"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"activitypub_federation",
"actix-cors",
@ -3438,7 +3438,7 @@ dependencies = [
[[package]]
name = "lemmy_utils"
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
dependencies = [
"actix-web",
"anyhow",

View file

@ -1,5 +1,5 @@
[workspace.package]
version = "0.19.4-beta.6"
version = "0.19.4-beta.7"
edition = "2021"
description = "A link aggregator for the fediverse"
license = "AGPL-3.0"
@ -88,18 +88,18 @@ unused_self = "deny"
unwrap_used = "deny"
[workspace.dependencies]
lemmy_api = { version = "=0.19.4-beta.6", path = "./crates/api" }
lemmy_api_crud = { version = "=0.19.4-beta.6", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.19.4-beta.6", path = "./crates/apub" }
lemmy_utils = { version = "=0.19.4-beta.6", path = "./crates/utils", default-features = false }
lemmy_db_schema = { version = "=0.19.4-beta.6", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.19.4-beta.6", path = "./crates/api_common" }
lemmy_routes = { version = "=0.19.4-beta.6", path = "./crates/routes" }
lemmy_db_views = { version = "=0.19.4-beta.6", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.19.4-beta.6", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.19.4-beta.6", path = "./crates/db_views_moderator" }
lemmy_federate = { version = "=0.19.4-beta.6", path = "./crates/federate" }
activitypub_federation = { version = "0.5.5", default-features = false, features = [
lemmy_api = { version = "=0.19.4-beta.7", path = "./crates/api" }
lemmy_api_crud = { version = "=0.19.4-beta.7", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.19.4-beta.7", path = "./crates/apub" }
lemmy_utils = { version = "=0.19.4-beta.7", path = "./crates/utils", default-features = false }
lemmy_db_schema = { version = "=0.19.4-beta.7", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.19.4-beta.7", path = "./crates/api_common" }
lemmy_routes = { version = "=0.19.4-beta.7", path = "./crates/routes" }
lemmy_db_views = { version = "=0.19.4-beta.7", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.19.4-beta.7", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.19.4-beta.7", path = "./crates/db_views_moderator" }
lemmy_federate = { version = "=0.19.4-beta.7", path = "./crates/federate" }
activitypub_federation = { version = "0.5.6", default-features = false, features = [
"actix-web",
] }
diesel = "2.1.6"

1
api_tests/.npmrc Normal file
View file

@ -0,0 +1 @@
package-manager-strict=false

File diff suppressed because it is too large Load diff

View file

@ -40,7 +40,7 @@ pub struct Register {
pub username: String,
pub password: Sensitive<String>,
pub password_verify: Sensitive<String>,
pub show_nsfw: bool,
pub show_nsfw: Option<bool>,
/// email is mandatory if email verification is enabled on the server
pub email: Option<Sensitive<String>>,
/// The UUID of the captcha item.

View file

@ -142,12 +142,17 @@ pub async fn register(
.map(|lang_str| lang_str.split('-').next().unwrap_or_default().to_string())
.collect();
// Show nsfw content if param is true, or if content_warning exists
let show_nsfw = data
.show_nsfw
.unwrap_or(site_view.site.content_warning.is_some());
// Create the local user
let local_user_form = LocalUserInsertForm::builder()
.person_id(inserted_person.id)
.email(data.email.as_deref().map(str::to_lowercase))
.password_encrypted(data.password.to_string())
.show_nsfw(Some(data.show_nsfw))
.show_nsfw(Some(show_nsfw))
.accepted_application(accepted_application)
.default_listing_type(Some(local_site.default_post_listing_type))
.post_listing_mode(Some(local_site.default_post_listing_mode))

View file

@ -0,0 +1,49 @@
{
"@context": ["https://www.w3.org/ns/activitystreams"],
"id": "https://pfefferle.org/lemmy-part-4/#activity#activity",
"type": "Announce",
"audience": "https://pfefferle.org/@pfefferle.org",
"published": "2024-05-03T12:32:29Z",
"updated": "2024-05-06T08:20:33Z",
"to": [
"https://www.w3.org/ns/activitystreams#Public",
"https://pfefferle.org/wp-json/activitypub/1.0/actors/1/followers"
],
"cc": [],
"object": {
"id": "https://pfefferle.org/lemmy-part-4/#activity",
"type": "Update",
"audience": "https://pfefferle.org/@pfefferle.org",
"published": "2024-05-03T12:32:29Z",
"updated": "2024-05-06T08:20:33Z",
"to": [
"https://www.w3.org/ns/activitystreams#Public",
"https://pfefferle.org/wp-json/activitypub/1.0/actors/1/followers"
],
"cc": [],
"object": {
"id": "https://pfefferle.org/lemmy-part-4/",
"type": "Article",
"attachment": [],
"attributedTo": "https://pfefferle.org/author/pfefferle/",
"audience": "https://pfefferle.org/@pfefferle.org",
"content": "\u003Cp\u003EIdentifies one or more entities that represent the total population of entities for which the object can considered to be relevant. Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant. \u003C/p\u003E",
"contentMap": {
"en": "\u003Cp\u003EIdentifies one or more entities that represent the total population of entities for which the object can considered to be relevant. Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant. \u003C/p\u003E"
},
"name": "Lemmy (Part 4)",
"published": "2024-05-03T12:32:29Z",
"summary": "Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant. Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant.Identifies one or more entities that represent the total population of entities for which the object [...]",
"tag": [],
"updated": "2024-05-06T08:20:33Z",
"url": "https://pfefferle.org/lemmy-part-4/",
"to": [
"https://www.w3.org/ns/activitystreams#Public",
"https://pfefferle.org/wp-json/activitypub/1.0/actors/1/followers"
],
"cc": []
},
"actor": "https://pfefferle.org/author/pfefferle/"
},
"actor": "https://pfefferle.org/@pfefferle.org"
}

View file

@ -0,0 +1,66 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
"https://purl.archive.org/socialweb/webfinger",
{
"schema": "http://schema.org#",
"toot": "http://joinmastodon.org/ns#",
"webfinger": "https://webfinger.net/#",
"lemmy": "https://join-lemmy.org/ns#",
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"Hashtag": "as:Hashtag",
"featured": {
"@id": "toot:featured",
"@type": "@id"
},
"featuredTags": {
"@id": "toot:featuredTags",
"@type": "@id"
},
"moderators": {
"@id": "lemmy:moderators",
"@type": "@id"
},
"postingRestrictedToMods": "lemmy:postingRestrictedToMods",
"discoverable": "toot:discoverable",
"indexable": "toot:indexable",
"resource": "webfinger:resource"
}
],
"id": "https://pfefferle.org/@pfefferle.org",
"type": "Group",
"attachment": [],
"attributedTo": "https://pfefferle.org/wp-json/activitypub/1.0/collections/moderators",
"name": "Matthias Pfefferle",
"icon": {
"type": "Image",
"url": "https://pfefferle.org/wp-content/uploads/2023/06/cropped-BeLItBV-_400x400.jpg"
},
"published": "2024-04-03T16:58:22Z",
"summary": "<p>Webworker, blogger und podcaster</p>\n",
"tag": [],
"url": "https://pfefferle.org/@pfefferle.org",
"inbox": "https://pfefferle.org/wp-json/activitypub/1.0/users/0/inbox",
"outbox": "https://pfefferle.org/wp-json/activitypub/1.0/users/0/outbox",
"following": "https://pfefferle.org/wp-json/activitypub/1.0/users/0/following",
"followers": "https://pfefferle.org/wp-json/activitypub/1.0/users/0/followers",
"preferredUsername": "pfefferle.org",
"endpoints": {
"sharedInbox": "https://pfefferle.org/wp-json/activitypub/1.0/inbox"
},
"publicKey": {
"id": "https://pfefferle.org/@pfefferle.org#main-key",
"owner": "https://pfefferle.org/@pfefferle.org",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuq8xeLMFcaCwPFBhgMRE\n/dDh2XKoNXFXnixctmK8BXSuuLMxucm3I/8NyhIvb3LqU+uP1fO8F0ecUbk2sN+x\nKag5vIV6yKXzJ8ILMWQ9AaELpXDmMZqL0zal0LUJRAOkDgPDovDAoq6tx++yDoV0\njdVbf9CoZKit1cz2ZrEuE5dswq3J/z9+c6POkhCkWEX5TPJzkOrmnjkvrXxGHUJ2\nA3+P+VaZhd5cmvqYosSpYNJshxCdev12pIF78OnYLiYiyXlgGHU+7uQR0M4tTcij\n6cUdLkms9m+b6H3ctXntPn410e5YLFPldjAYzQB5wHVdFZsWtyrbqfYdCa+KkKpA\nvwIDAQAB\n-----END PUBLIC KEY-----\n"
},
"manuallyApprovesFollowers": false,
"featured": "https://pfefferle.org/wp-json/activitypub/1.0/users/0/collections/featured",
"moderators": "https://pfefferle.org/wp-json/activitypub/1.0/collections/moderators",
"discoverable": true,
"indexable": true,
"webfinger": "pfefferle.org@pfefferle.org",
"postingRestrictedToMods": true
}

View file

@ -0,0 +1,24 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"Hashtag": "as:Hashtag"
}
],
"id": "https://pfefferle.org?c=148",
"type": "Note",
"attributedTo": "https://pfefferle.org/author/pfefferle/",
"content": "<p>Nice! Hello from WordPress!</p>",
"contentMap": {
"en": "<p>Nice! Hello from WordPress!</p>"
},
"inReplyTo": "https://socialhub.activitypub.rocks/ap/object/ce040f1ead95964f6dbbf1084b81432d",
"published": "2024-04-30T15:21:13Z",
"tag": [],
"url": "https://pfefferle.org?c=148",
"to": [
"https://www.w3.org/ns/activitystreams#Public",
"https://pfefferle.org/wp-json/activitypub/1.0/users/0/followers"
],
"cc": []
}

View file

@ -0,0 +1,26 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"Hashtag": "as:Hashtag"
}
],
"id": "https://pfefferle.org/this-is-a-test-federation/",
"type": "Article",
"attachment": [],
"attributedTo": "https://pfefferle.org/author/pfefferle/",
"content": "<p>with Discource!</p>",
"contentMap": {
"en": "<p>with Discource!</p>"
},
"name": "This is a test-federation",
"published": "2024-04-30T15:16:41Z",
"summary": "with Discource! [...]",
"tag": [],
"url": "https://pfefferle.org/this-is-a-test-federation/",
"to": [
"https://www.w3.org/ns/activitystreams#Public",
"https://pfefferle.org/wp-json/activitypub/1.0/users/1/followers"
],
"cc": []
}

View file

@ -0,0 +1,74 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
"https://purl.archive.org/socialweb/webfinger",
{
"schema": "http://schema.org#",
"toot": "http://joinmastodon.org/ns#",
"webfinger": "https://webfinger.net/#",
"lemmy": "https://join-lemmy.org/ns#",
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"Hashtag": "as:Hashtag",
"featured": {
"@id": "toot:featured",
"@type": "@id"
},
"featuredTags": {
"@id": "toot:featuredTags",
"@type": "@id"
},
"moderators": {
"@id": "lemmy:moderators",
"@type": "@id"
},
"postingRestrictedToMods": "lemmy:postingRestrictedToMods",
"discoverable": "toot:discoverable",
"indexable": "toot:indexable",
"resource": "webfinger:resource"
}
],
"id": "https://pfefferle.org/author/pfefferle/",
"type": "Person",
"attachment": [
{
"type": "PropertyValue",
"name": "Blog",
"value": "<a rel=\"me\" title=\"https://pfefferle.org/\" target=\"_blank\" href=\"https://pfefferle.org/\">pfefferle.org</a>"
},
{
"type": "PropertyValue",
"name": "Profile",
"value": "<a rel=\"me\" title=\"https://pfefferle.org/author/pfefferle/\" target=\"_blank\" href=\"https://pfefferle.org/author/pfefferle/\">pfefferle.org</a>"
}
],
"name": "Matthias Pfefferle",
"icon": {
"type": "Image",
"url": "https://secure.gravatar.com/avatar/a2bdca7870e859658cece96c044b3be5?s=120&#038;d=mm&#038;r=g"
},
"published": "2014-02-10T15:23:08Z",
"summary": "<p>Ich arbeite als Open Web Lead für Automattic.</p>\n",
"tag": [],
"url": "https://pfefferle.org/author/pfefferle/",
"inbox": "https://pfefferle.org/wp-json/activitypub/1.0/users/1/inbox",
"outbox": "https://pfefferle.org/wp-json/activitypub/1.0/users/1/outbox",
"following": "https://pfefferle.org/wp-json/activitypub/1.0/users/1/following",
"followers": "https://pfefferle.org/wp-json/activitypub/1.0/users/1/followers",
"preferredUsername": "matthias",
"endpoints": {
"sharedInbox": "https://pfefferle.org/wp-json/activitypub/1.0/inbox"
},
"publicKey": {
"id": "https://pfefferle.org/author/pfefferle/#main-key",
"owner": "https://pfefferle.org/author/pfefferle/",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvTA5RA40nOsso04RSwyX\nHXTojRPUMlIlArDcSy3M5GUJp9/xbxSUOdBjqd31KKB1GIi3vrLmD1Qi/ZqS95Qy\nw2Zd3xOsCg+o9bsyOG+O6Y8Lu+HEB5JKLUbNHdiSviakJ8wGadH9Wm4WIiN20y+q\n/u6lgxgiWfZ2CFCN6SOc28fUKi9NmKvXK+M12BhFfy1tC5KWXKDm0UbfI1+dmqhR\n3Ffe6vEsCI/YIVVdWxQ9kouOd0XSHOGdslktkepRO7IP9i9TdwyeCa0WWRoeO5Wa\ntVpc1Y0WuNbTM2ksIXTg0G+rO1/6KO/hrHnGu3RCfb/ZIHK5L/aWYb9B3PG3LyKV\n+wIDAQAB\n-----END PUBLIC KEY-----\n"
},
"manuallyApprovesFollowers": false,
"featured": "https://pfefferle.org/wp-json/activitypub/1.0/users/1/collections/featured",
"discoverable": true,
"indexable": true,
"webfinger": "matthias@pfefferle.org"
}

View file

@ -20,7 +20,8 @@ use lemmy_db_schema::{
};
use lemmy_utils::error::{LemmyErrorType, LemmyResult};
use serde::{Deserialize, Serialize};
use std::ops::Deref;
use std::{ops::Deref, time::Duration};
use tokio::time::timeout;
use url::Url;
mod comment;
@ -30,13 +31,22 @@ mod post;
pub mod routes;
pub mod site;
const INCOMING_ACTIVITY_TIMEOUT: Duration = Duration::from_secs(9);
pub async fn shared_inbox(
request: HttpRequest,
body: Bytes,
data: Data<LemmyContext>,
) -> LemmyResult<HttpResponse> {
receive_activity::<SharedInboxActivities, UserOrCommunity, LemmyContext>(request, body, &data)
let receive_fut =
receive_activity::<SharedInboxActivities, UserOrCommunity, LemmyContext>(request, body, &data);
// Set a timeout shorter than `REQWEST_TIMEOUT` for processing incoming activities. This is to
// avoid taking a long time to process an incoming activity when a required data fetch times out.
// In this case our own instance would timeout and be marked as dead by the sender. Better to
// consider the activity broken and move on.
timeout(INCOMING_ACTIVITY_TIMEOUT, receive_fut)
.await
.map_err(|_| LemmyErrorType::InboxTimeout)?
}
/// Convert the data to json and turn it into an HTTP Response with the correct ActivityPub

View file

@ -100,7 +100,7 @@ impl Object for ApubSite {
kind: ApplicationType::Application,
id: self.id().into(),
name: self.name.clone(),
preferred_username: data.domain().to_string(),
preferred_username: Some(data.domain().to_string()),
content: self.sidebar.as_ref().map(|d| markdown_to_html(d)),
source: self.sidebar.clone().map(Source::new),
summary: self.description.clone(),

View file

@ -96,4 +96,10 @@ mod tests {
test_json::<Report>("assets/mbin/activities/flag.json")?;
Ok(())
}
#[test]
fn test_parse_wordpress_activities() -> LemmyResult<()> {
test_json::<AnnounceActivity>("assets/wordpress/activities/announce.json")?;
Ok(())
}
}

View file

@ -22,7 +22,7 @@ pub struct Instance {
/// site name
pub(crate) name: String,
/// instance domain, necessary for mastodon authorized fetch
pub(crate) preferred_username: String,
pub(crate) preferred_username: Option<String>,
pub(crate) inbox: Url,
/// mandatory field in activitypub, lemmy currently serves an empty outbox
pub(crate) outbox: Url,

View file

@ -206,4 +206,13 @@ mod tests {
test_json::<Person>("assets/nodebb/objects/person.json")?;
Ok(())
}
#[test]
fn test_parse_object_wordpress() -> LemmyResult<()> {
test_json::<Group>("assets/wordpress/objects/group.json")?;
test_json::<Page>("assets/wordpress/objects/page.json")?;
test_json::<Person>("assets/wordpress/objects/person.json")?;
test_json::<Note>("assets/wordpress/objects/note.json")?;
Ok(())
}
}

View file

@ -233,6 +233,10 @@ impl ActivityHandler for Page {
#[async_trait::async_trait]
impl InCommunity for Page {
async fn community(&self, context: &Data<LemmyContext>) -> LemmyResult<ApubCommunity> {
if let Some(audience) = &self.audience {
return audience.dereference(context).await;
}
let community = match &self.attributed_to {
AttributedTo::Lemmy(_) => {
let mut iter = self.to.iter().merge(self.cc.iter());
@ -243,7 +247,7 @@ impl InCommunity for Page {
break c;
}
} else {
Err(LemmyErrorType::NoCommunityFoundInCc)?
Err(LemmyErrorType::CouldntFindCommunity)?;
}
}
}
@ -251,11 +255,12 @@ impl InCommunity for Page {
p.iter()
.find(|a| a.kind == PersonOrGroupType::Group)
.map(|a| ObjectId::<ApubCommunity>::from(a.id.clone().into_inner()))
.ok_or(LemmyErrorType::PageDoesNotSpecifyGroup)?
.ok_or(LemmyErrorType::CouldntFindCommunity)?
.dereference(context)
.await?
}
};
if let Some(audience) = &self.audience {
verify_community_matches(audience, community.actor_id.clone())?;
}

View file

@ -5,6 +5,12 @@
-- (even if only other columns are updated) because triggers can run after the deletion of referenced rows and
-- before the automatic deletion of the row that references it. This is not a problem for insert or delete.
--
-- After a row update begins, a concurrent update on the same row can't begin until the whole
-- transaction that contains the first update is finished. To reduce this locking, statements in
-- triggers should be ordered based on the likelihood of concurrent writers. For example, updating
-- site_aggregates should be done last because the same row is updated for all local stuff. If
-- it were not last, then the locking period for concurrent writers would extend to include the
-- time consumed by statements that come after.
--
--
-- Create triggers for both post and comments
@ -38,16 +44,18 @@ BEGIN
(thing_like).thing_id, coalesce(sum(count_diff) FILTER (WHERE (thing_like).score = 1), 0) AS upvotes, coalesce(sum(count_diff) FILTER (WHERE (thing_like).score != 1), 0) AS downvotes FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (thing_like).thing_id) AS diff
WHERE
a.thing_id = diff.thing_id
RETURNING
r.creator_id_from_thing_aggregates (a.*) AS creator_id, diff.upvotes - diff.downvotes AS score)
UPDATE
person_aggregates AS a
SET
thing_score = a.thing_score + diff.score FROM (
SELECT
creator_id, sum(score) AS score FROM thing_diff GROUP BY creator_id) AS diff
WHERE
a.person_id = diff.creator_id;
AND (diff.upvotes, diff.downvotes) != (0, 0)
RETURNING
r.creator_id_from_thing_aggregates (a.*) AS creator_id, diff.upvotes - diff.downvotes AS score)
UPDATE
person_aggregates AS a
SET
thing_score = a.thing_score + diff.score FROM (
SELECT
creator_id, sum(score) AS score FROM thing_diff GROUP BY creator_id) AS diff
WHERE
a.person_id = diff.creator_id
AND diff.score != 0;
RETURN NULL;
END;
$$);
@ -62,6 +70,21 @@ CALL r.post_or_comment ('post');
CALL r.post_or_comment ('comment');
-- Create triggers that update counts in parent aggregates
CREATE FUNCTION r.parent_comment_ids (path ltree)
RETURNS SETOF int
LANGUAGE sql
IMMUTABLE parallel safe
BEGIN
ATOMIC
SELECT
comment_id::int
FROM
string_to_table (ltree2text (path), '.') AS comment_id
-- Skip first and last
LIMIT (nlevel (path) - 2) OFFSET 1;
END;
CALL r.create_triggers ('comment', $$
BEGIN
UPDATE
@ -76,60 +99,84 @@ BEGIN
r.is_counted (comment)
GROUP BY (comment).creator_id) AS diff
WHERE
a.person_id = diff.creator_id;
a.person_id = diff.creator_id
AND diff.comment_count != 0;
UPDATE
site_aggregates AS a
comment_aggregates AS a
SET
comments = a.comments + diff.comments
child_count = a.child_count + diff.child_count
FROM (
SELECT
coalesce(sum(count_diff), 0) AS comments
FROM
select_old_and_new_rows AS old_and_new_rows
WHERE
r.is_counted (comment)
AND (comment).local) AS diff;
parent_id,
coalesce(sum(count_diff), 0) AS child_count
FROM (
-- For each inserted or deleted comment, this outputs 1 row for each parent comment.
-- For example, this:
--
-- count_diff | (comment).path
-- ------------+----------------
-- 1 | 0.5.6.7
-- 1 | 0.5.6.7.8
--
-- becomes this:
--
-- count_diff | parent_id
-- ------------+-----------
-- 1 | 5
-- 1 | 6
-- 1 | 5
-- 1 | 6
-- 1 | 7
SELECT
count_diff,
parent_id
FROM
select_old_and_new_rows AS old_and_new_rows,
LATERAL r.parent_comment_ids ((comment).path) AS parent_id) AS expanded_old_and_new_rows
GROUP BY
parent_id) AS diff
WHERE
a.comment_id = diff.parent_id
AND diff.child_count != 0;
WITH post_diff AS (
UPDATE
post_aggregates AS a
SET
comments = a.comments + diff.comments,
newest_comment_time = GREATEST (a.newest_comment_time, (
SELECT
published
FROM select_new_rows AS new_comment
WHERE
a.post_id = new_comment.post_id ORDER BY published DESC LIMIT 1)),
newest_comment_time_necro = GREATEST (a.newest_comment_time_necro, (
SELECT
published
FROM select_new_rows AS new_comment
WHERE
a.post_id = new_comment.post_id
-- Ignore comments from the post's creator
AND a.creator_id != new_comment.creator_id
-- Ignore comments on old posts
AND a.published > (new_comment.published - '2 days'::interval)
ORDER BY published DESC LIMIT 1))
newest_comment_time = GREATEST (a.newest_comment_time, diff.newest_comment_time),
newest_comment_time_necro = GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro)
FROM (
SELECT
(comment).post_id,
coalesce(sum(count_diff), 0) AS comments
post.id AS post_id,
coalesce(sum(count_diff), 0) AS comments,
-- Old rows are excluded using `count_diff = 1`
max((comment).published) FILTER (WHERE count_diff = 1) AS newest_comment_time,
max((comment).published) FILTER (WHERE count_diff = 1
-- Ignore comments from the post's creator
AND post.creator_id != (comment).creator_id
-- Ignore comments on old posts
AND post.published > ((comment).published - '2 days'::interval)) AS newest_comment_time_necro,
r.is_counted (post.*) AS include_in_community_aggregates
FROM
select_old_and_new_rows AS old_and_new_rows
LEFT JOIN post ON post.id = (comment).post_id
WHERE
r.is_counted (comment)
GROUP BY
(comment).post_id) AS diff
LEFT JOIN post ON post.id = diff.post_id
post.id) AS diff
WHERE
a.post_id = diff.post_id
AND (diff.comments,
GREATEST (a.newest_comment_time, diff.newest_comment_time),
GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro)) != (0,
a.newest_comment_time,
a.newest_comment_time_necro)
RETURNING
a.community_id,
diff.comments,
r.is_counted (post.*) AS include_in_community_aggregates)
diff.include_in_community_aggregates)
UPDATE
community_aggregates AS a
SET
@ -145,7 +192,23 @@ FROM (
GROUP BY
community_id) AS diff
WHERE
a.community_id = diff.community_id;
a.community_id = diff.community_id
AND diff.comments != 0;
UPDATE
site_aggregates AS a
SET
comments = a.comments + diff.comments
FROM (
SELECT
coalesce(sum(count_diff), 0) AS comments
FROM
select_old_and_new_rows AS old_and_new_rows
WHERE
r.is_counted (comment)
AND (comment).local) AS diff
WHERE
diff.comments != 0;
RETURN NULL;
@ -167,20 +230,8 @@ BEGIN
r.is_counted (post)
GROUP BY (post).creator_id) AS diff
WHERE
a.person_id = diff.creator_id;
UPDATE
site_aggregates AS a
SET
posts = a.posts + diff.posts
FROM (
SELECT
coalesce(sum(count_diff), 0) AS posts
FROM
select_old_and_new_rows AS old_and_new_rows
WHERE
r.is_counted (post)
AND (post).local) AS diff;
a.person_id = diff.creator_id
AND diff.post_count != 0;
UPDATE
community_aggregates AS a
@ -197,7 +248,23 @@ FROM (
GROUP BY
(post).community_id) AS diff
WHERE
a.community_id = diff.community_id;
a.community_id = diff.community_id
AND diff.posts != 0;
UPDATE
site_aggregates AS a
SET
posts = a.posts + diff.posts
FROM (
SELECT
coalesce(sum(count_diff), 0) AS posts
FROM
select_old_and_new_rows AS old_and_new_rows
WHERE
r.is_counted (post)
AND (post).local) AS diff
WHERE
diff.posts != 0;
RETURN NULL;
@ -217,7 +284,9 @@ BEGIN
FROM select_old_and_new_rows AS old_and_new_rows
WHERE
r.is_counted (community)
AND (community).local) AS diff;
AND (community).local) AS diff
WHERE
diff.communities != 0;
RETURN NULL;
@ -235,7 +304,9 @@ BEGIN
SELECT
coalesce(sum(count_diff), 0) AS users
FROM select_old_and_new_rows AS old_and_new_rows
WHERE (person).local) AS diff;
WHERE (person).local) AS diff
WHERE
diff.users != 0;
RETURN NULL;
@ -270,7 +341,8 @@ BEGIN
GROUP BY
old_post.community_id) AS diff
WHERE
a.community_id = diff.community_id;
a.community_id = diff.community_id
AND diff.comments != 0;
RETURN NULL;
END;
$$;
@ -296,7 +368,8 @@ BEGIN
LEFT JOIN community ON community.id = (community_follower).community_id
LEFT JOIN person ON person.id = (community_follower).person_id GROUP BY (community_follower).community_id) AS diff
WHERE
a.community_id = diff.community_id;
a.community_id = diff.community_id
AND (diff.subscribers, diff.subscribers_local) != (0, 0);
RETURN NULL;
@ -474,3 +547,24 @@ CREATE TRIGGER delete_follow
FOR EACH ROW
EXECUTE FUNCTION r.delete_follow_before_person ();
-- Triggers that change values before insert or update
CREATE FUNCTION r.comment_change_values ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
id text = NEW.id::text;
BEGIN
-- Make `path` end with `id` if it doesn't already
IF NOT (NEW.path ~ ('*.' || id)::lquery) THEN
NEW.path = NEW.path || id;
END IF;
RETURN NEW;
END
$$;
CREATE TRIGGER change_values
BEFORE INSERT OR UPDATE ON comment
FOR EACH ROW
EXECUTE FUNCTION r.comment_change_values ();

View file

@ -15,12 +15,7 @@ use crate::{
utils::{functions::coalesce, get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT},
};
use chrono::{DateTime, Utc};
use diesel::{
dsl::{insert_into, sql_query},
result::Error,
ExpressionMethods,
QueryDsl,
};
use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl};
use diesel_async::RunQueryDsl;
use diesel_ltree::Ltree;
use url::Url;
@ -72,81 +67,23 @@ impl Comment {
parent_path: Option<&Ltree>,
) -> Result<Comment, Error> {
let conn = &mut get_conn(pool).await?;
let comment_form = (comment_form, parent_path.map(|p| comment::path.eq(p)));
conn
.build_transaction()
.run(|conn| {
Box::pin(async move {
// Insert, to get the id
let inserted_comment = if let Some(timestamp) = timestamp {
insert_into(comment::table)
.values(comment_form)
.on_conflict(comment::ap_id)
.filter_target(coalesce(comment::updated, comment::published).lt(timestamp))
.do_update()
.set(comment_form)
.get_result::<Self>(conn)
.await?
} else {
insert_into(comment::table)
.values(comment_form)
.get_result::<Self>(conn)
.await?
};
let comment_id = inserted_comment.id;
// You need to update the ltree column
let ltree = Ltree(if let Some(parent_path) = parent_path {
// The previous parent will already have 0 in it
// Append this comment id
format!("{}.{}", parent_path.0, comment_id)
} else {
// '0' is always the first path, append to that
format!("{}.{}", 0, comment_id)
});
let updated_comment = diesel::update(comment::table.find(comment_id))
.set(comment::path.eq(ltree))
.get_result::<Self>(conn)
.await?;
// Update the child count for the parent comment_aggregates
// You could do this with a trigger, but since you have to do this manually anyway,
// you can just have it here
if let Some(parent_path) = parent_path {
// You have to update counts for all parents, not just the immediate one
// TODO if the performance of this is terrible, it might be better to do this as part of a
// scheduled query... although the counts would often be wrong.
//
// The child_count query for reference:
// select c.id, c.path, count(c2.id) as child_count from comment c
// left join comment c2 on c2.path <@ c.path and c2.path != c.path
// group by c.id
let parent_id = parent_path.0.split('.').nth(1);
if let Some(parent_id) = parent_id {
let top_parent = format!("0.{}", parent_id);
let update_child_count_stmt = format!(
"
update comment_aggregates ca set child_count = c.child_count
from (
select c.id, c.path, count(c2.id) as child_count from comment c
join comment c2 on c2.path <@ c.path and c2.path != c.path
and c.path <@ '{top_parent}'
group by c.id
) as c
where ca.comment_id = c.id"
);
sql_query(update_child_count_stmt).execute(conn).await?;
}
}
Ok(updated_comment)
}) as _
})
.await
if let Some(timestamp) = timestamp {
insert_into(comment::table)
.values(comment_form)
.on_conflict(comment::ap_id)
.filter_target(coalesce(comment::updated, comment::published).lt(timestamp))
.do_update()
.set(comment_form)
.get_result::<Self>(conn)
.await
} else {
insert_into(comment::table)
.values(comment_form)
.get_result::<Self>(conn)
.await
}
}
pub async fn read_from_apub_id(

View file

@ -43,21 +43,18 @@ async fn node_info(context: web::Data<LemmyContext>) -> Result<HttpResponse, Err
.map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?
.ok_or(ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
let protocols = if site_view.local_site.federation_enabled {
Some(vec!["activitypub".to_string()])
} else {
None
};
// Since there are 3 registration options,
// we need to set open_registrations as true if RegistrationMode is not Closed.
let open_registrations = Some(site_view.local_site.registration_mode != RegistrationMode::Closed);
let json = NodeInfo {
version: Some("2.0".to_string()),
version: Some("2.1".to_string()),
software: Some(NodeInfoSoftware {
name: Some("lemmy".to_string()),
version: Some(VERSION.to_string()),
repository: Some("https://github.com/LemmyNet/lemmy".to_string()),
homepage: Some("https://join-lemmy.org/".to_string()),
}),
protocols,
protocols: Some(vec!["activitypub".to_string()]),
usage: Some(NodeInfoUsage {
users: Some(NodeInfoUsers {
total: Some(site_view.counts.users),
@ -68,6 +65,11 @@ async fn node_info(context: web::Data<LemmyContext>) -> Result<HttpResponse, Err
local_comments: Some(site_view.counts.comments),
}),
open_registrations,
services: Some(NodeInfoServices {
inbound: Some(vec![]),
outbound: Some(vec![]),
}),
metadata: Some(vec![]),
};
Ok(HttpResponse::Ok().json(json))
@ -84,6 +86,7 @@ struct NodeInfoWellKnownLinks {
pub href: Url,
}
/// Nodeinfo spec: http://nodeinfo.diaspora.software/docson/index.html#/ns/schema/2.1
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase", default)]
pub struct NodeInfo {
@ -92,6 +95,9 @@ pub struct NodeInfo {
pub protocols: Option<Vec<String>>,
pub usage: Option<NodeInfoUsage>,
pub open_registrations: Option<bool>,
/// These fields are required by the spec for no reason
pub services: Option<NodeInfoServices>,
pub metadata: Option<Vec<String>>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
@ -99,6 +105,8 @@ pub struct NodeInfo {
pub struct NodeInfoSoftware {
pub name: Option<String>,
pub version: Option<String>,
pub repository: Option<String>,
pub homepage: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
@ -116,3 +124,10 @@ pub struct NodeInfoUsers {
pub active_halfyear: Option<i64>,
pub active_month: Option<i64>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase", default)]
pub struct NodeInfoServices {
pub inbound: Option<Vec<String>>,
pub outbound: Option<Vec<String>>,
}

View file

@ -99,8 +99,6 @@ pub enum LemmyErrorType {
PersonIsBannedFromSite(String),
InvalidVoteValue,
PageDoesNotSpecifyCreator,
PageDoesNotSpecifyGroup,
NoCommunityFoundInCc,
NoEmailSetup,
LocalSiteNotSetup,
EmailSmtpServerNeedsAPort,
@ -178,6 +176,7 @@ pub enum LemmyErrorType {
CantBlockLocalInstance,
UrlWithoutDomain,
PluginError(String),
InboxTimeout,
Unknown(String),
}

@ -1 +1 @@
Subproject commit a4681f70a4ddf077951ed2dcc8cf90bb243d4828
Subproject commit f0ab81deea347c433277a90ae752b10f68473719

View file

@ -0,0 +1,3 @@
SELECT
1;

View file

@ -0,0 +1,4 @@
-- This migration exists to trigger re-execution of replaceable_schema
SELECT
1;

View file

@ -4,39 +4,39 @@ set -e
echo "Do not stop in the middle of this upgrade, wait until you see the message: Upgrade complete."
echo "Stopping lemmy and all services..."
sudo docker-compose stop
sudo docker compose stop
echo "Make sure postgres is started..."
sudo docker-compose up -d postgres
sudo docker compose up -d postgres
echo "Waiting..."
sleep 20s
echo "Exporting the Database to 15_16.dump.sql ..."
sudo docker-compose exec -T postgres pg_dumpall -c -U lemmy > 15_16_dump.sql
sudo docker compose exec -T postgres pg_dumpall -c -U lemmy | sudo tee 15_16_dump.sql > /dev/null
echo "Done."
echo "Stopping postgres..."
sudo docker-compose stop postgres
sudo docker compose stop postgres
echo "Waiting..."
sleep 20s
echo "Removing the old postgres folder"
sudo rm -rf volumes/postgres
echo "Updating docker-compose to use postgres version 16."
sed -i "s/image: postgres:.*/image: postgres:16-alpine/" ./docker-compose.yml
echo "Updating docker compose to use postgres version 16."
sudo sed -i "s/image: .*postgres:.*/image: docker.io/postgres:16-alpine/" ./docker-compose.yml
echo "Starting up new postgres..."
sudo docker-compose up -d postgres
sudo docker compose up -d postgres
echo "Waiting..."
sleep 20s
echo "Importing the database...."
cat 15_16_dump.sql | sudo docker-compose exec -T postgres psql -U lemmy
sudo cat 15_16_dump.sql | sudo docker compose exec -T postgres psql -U lemmy
echo "Done."
echo "Starting up lemmy..."
sudo docker-compose up -d
sudo docker compose up -d
echo "A copy of your old database is at 15_16.dump.sql . You can delete this file if the upgrade went smoothly."
echo "Upgrade complete."

View file

@ -161,10 +161,10 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> {
rate_limit_cell.clone(),
);
if !args.disable_scheduled_tasks {
let scheduled_tasks = (!args.disable_scheduled_tasks).then(|| {
// Schedules various cleanup tasks for the DB
let _scheduled_tasks = tokio::task::spawn(scheduled_tasks::setup(context.clone()));
}
tokio::task::spawn(scheduled_tasks::setup(context.clone()))
});
if let Some(prometheus) = SETTINGS.prometheus.clone() {
serve_prometheus(prometheus, context.clone())?;
@ -219,7 +219,7 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> {
let mut interrupt = tokio::signal::unix::signal(SignalKind::interrupt())?;
let mut terminate = tokio::signal::unix::signal(SignalKind::terminate())?;
if server.is_some() || federate.is_some() {
if server.is_some() || federate.is_some() || scheduled_tasks.is_some() {
tokio::select! {
_ = tokio::signal::ctrl_c() => {
tracing::warn!("Received ctrl-c, shutting down gracefully...");

View file

@ -77,7 +77,6 @@ where
}
let mut res = svc.call(service_req).await?;
// TODO: add after hook
let after_plugin_hook = format!("api_after_{method}_{path}").to_lowercase();
info!("Calling plugin hook {}", &after_plugin_hook);
if let Some(mut plugins) = load_plugins()? {