Adding instance software and version. Fixes #2222 (#2733)

* Adding instance software and version. Fixes #2222

* Fix clippy.

* Fix clippy 2

* Fixing some more issues.

Cargo.toml

@@ -71,7 +71,7 @@ tracing-error = "0.2.0"
 tracing-log = "0.1.3"
 tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
 url = { version = "2.3.1", features = ["serde"] }
-reqwest = { version = "0.11.12", features = ["json"] }
+reqwest = { version = "0.11.12", features = ["json", "blocking"] }
 reqwest-middleware = "0.2.0"
 reqwest-tracing = "0.4.0"
 clokwerk = "0.3.5"

crates/api_common/src/site.rs

@@ -1,7 +1,12 @@
 use crate::sensitive::Sensitive;
 use lemmy_db_schema::{
   newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId},
-  source::{language::Language, local_site::RegistrationMode, tagline::Tagline},
+  source::{
+    instance::Instance,
+    language::Language,
+    local_site::RegistrationMode,
+    tagline::Tagline,
+  },
   ListingType,
   ModlogActionType,
   SearchType,
@@ -239,9 +244,9 @@ pub struct LeaveAdmin {
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct FederatedInstances {
-  pub linked: Vec<String>,
-  pub allowed: Option<Vec<String>>,
-  pub blocked: Option<Vec<String>>,
+  pub linked: Vec<Instance>,
+  pub allowed: Option<Vec<Instance>>,
+  pub blocked: Option<Vec<Instance>>,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]

crates/apub/src/lib.rs

@@ -114,13 +114,13 @@ fn check_apub_id_valid(
   }
 
   if let Some(blocked) = local_site_data.blocked_instances.as_ref() {
-    if blocked.contains(&domain) {
+    if blocked.iter().any(|i| domain.eq(&i.domain)) {
       return Err("Domain is blocked");
     }
   }
 
   if let Some(allowed) = local_site_data.allowed_instances.as_ref() {
-    if !allowed.contains(&domain) {
+    if !allowed.iter().any(|i| domain.eq(&i.domain)) {
       return Err("Domain is not in allowlist");
     }
   }
@@ -131,8 +131,8 @@ fn check_apub_id_valid(
 #[derive(Clone)]
 pub(crate) struct LocalSiteData {
   local_site: Option<LocalSite>,
-  allowed_instances: Option<Vec<String>>,
-  blocked_instances: Option<Vec<String>>,
+  allowed_instances: Option<Vec<Instance>>,
+  blocked_instances: Option<Vec<Instance>>,
 }
 
 pub(crate) async fn fetch_local_site_data(
@@ -175,7 +175,10 @@ pub(crate) fn check_apub_id_valid_with_strictness(
     if is_strict {
       // need to allow this explicitly because apub receive might contain objects from our local
       // instance.
-      let mut allowed_and_local = allowed.clone();
+      let mut allowed_and_local = allowed
+        .iter()
+        .map(|i| i.domain.clone())
+        .collect::<Vec<String>>();
       allowed_and_local.push(local_instance);
 
       if !allowed_and_local.contains(&domain) {

crates/db_schema/src/impls/federation_allowlist.rs

@@ -59,25 +59,24 @@ mod tests {
   #[serial]
   async fn test_allowlist_insert_and_clear() {
     let pool = &build_db_pool_for_tests().await;
-    let allowed = Some(vec![
+    let domains = vec![
       "tld1.xyz".to_string(),
       "tld2.xyz".to_string(),
       "tld3.xyz".to_string(),
-    ]);
+    ];
+
+    let allowed = Some(domains.clone());
 
     FederationAllowList::replace(pool, allowed).await.unwrap();
 
     let allows = Instance::allowlist(pool).await.unwrap();
+    let allows_domains = allows
+      .iter()
+      .map(|i| i.domain.clone())
+      .collect::<Vec<String>>();
 
     assert_eq!(3, allows.len());
-    assert_eq!(
-      vec![
-        "tld1.xyz".to_string(),
-        "tld2.xyz".to_string(),
-        "tld3.xyz".to_string()
-      ],
-      allows
-    );
+    assert_eq!(domains, allows_domains);
 
     // Now test clearing them via Some(empty vec)
     let clear_allows = Some(Vec::new());

crates/db_schema/src/impls/instance.rs

@@ -31,10 +31,10 @@ impl Instance {
     Self::create(pool, domain).await
   }
   pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result<Self, Error> {
-    let form = InstanceForm {
-      domain: domain.to_string(),
-      updated: Some(naive_now()),
-    };
+    let form = InstanceForm::builder()
+      .domain(domain.to_string())
+      .updated(Some(naive_now()))
+      .build();
     Self::create_from_form_conn(conn, &form).await
   }
   pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result<usize, Error> {
@@ -47,31 +47,31 @@ impl Instance {
     let conn = &mut get_conn(pool).await?;
     diesel::delete(instance::table).execute(conn).await
   }
-  pub async fn allowlist(pool: &DbPool) -> Result<Vec<String>, Error> {
+  pub async fn allowlist(pool: &DbPool) -> Result<Vec<Self>, Error> {
     let conn = &mut get_conn(pool).await?;
     instance::table
       .inner_join(federation_allowlist::table)
-      .select(instance::domain)
-      .load::<String>(conn)
+      .select(instance::all_columns)
+      .get_results(conn)
       .await
   }
 
-  pub async fn blocklist(pool: &DbPool) -> Result<Vec<String>, Error> {
+  pub async fn blocklist(pool: &DbPool) -> Result<Vec<Self>, Error> {
     let conn = &mut get_conn(pool).await?;
     instance::table
       .inner_join(federation_blocklist::table)
-      .select(instance::domain)
-      .load::<String>(conn)
+      .select(instance::all_columns)
+      .get_results(conn)
       .await
   }
 
-  pub async fn linked(pool: &DbPool) -> Result<Vec<String>, Error> {
+  pub async fn linked(pool: &DbPool) -> Result<Vec<Self>, Error> {
     let conn = &mut get_conn(pool).await?;
     instance::table
       .left_join(federation_blocklist::table)
       .filter(federation_blocklist::id.is_null())
-      .select(instance::domain)
-      .load::<String>(conn)
+      .select(instance::all_columns)
+      .get_results(conn)
       .await
   }
 }

crates/db_schema/src/schema.rs

@@ -648,6 +648,8 @@ table! {
   instance(id) {
     id -> Int4,
     domain -> Text,
+    software -> Nullable<Text>,
+    version -> Nullable<Text>,
     published -> Timestamp,
     updated -> Nullable<Timestamp>,
   }

crates/db_schema/src/source/instance.rs

@@ -1,21 +1,30 @@
 use crate::newtypes::InstanceId;
 #[cfg(feature = "full")]
 use crate::schema::instance;
+use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
+use typed_builder::TypedBuilder;
 
-#[derive(PartialEq, Eq, Debug)]
+#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
 #[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
 #[cfg_attr(feature = "full", diesel(table_name = instance))]
 pub struct Instance {
   pub id: InstanceId,
   pub domain: String,
+  pub software: Option<String>,
+  pub version: Option<String>,
   pub published: chrono::NaiveDateTime,
   pub updated: Option<chrono::NaiveDateTime>,
 }
 
+#[derive(Clone, TypedBuilder)]
+#[builder(field_defaults(default))]
 #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
 #[cfg_attr(feature = "full", diesel(table_name = instance))]
 pub struct InstanceForm {
+  #[builder(!default)]
   pub domain: String,
+  pub software: Option<String>,
+  pub version: Option<String>,
   pub updated: Option<chrono::NaiveDateTime>,
 }

crates/routes/src/nodeinfo.rs

@@ -34,27 +34,27 @@ async fn node_info(context: web::Data<LemmyContext>) -> Result<HttpResponse, Err
     .map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
 
   let protocols = if site_view.local_site.federation_enabled {
-    vec!["activitypub".to_string()]
+    Some(vec!["activitypub".to_string()])
   } else {
-    vec![]
+    None
   };
-  let open_registrations = site_view.local_site.registration_mode == RegistrationMode::Open;
+  let open_registrations = Some(site_view.local_site.registration_mode == RegistrationMode::Open);
   let json = NodeInfo {
-    version: "2.0".to_string(),
-    software: NodeInfoSoftware {
-      name: "lemmy".to_string(),
-      version: version::VERSION.to_string(),
-    },
+    version: Some("2.0".to_string()),
+    software: Some(NodeInfoSoftware {
+      name: Some("lemmy".to_string()),
+      version: Some(version::VERSION.to_string()),
+    }),
     protocols,
-    usage: NodeInfoUsage {
-      users: NodeInfoUsers {
-        total: site_view.counts.users,
-        active_halfyear: site_view.counts.users_active_half_year,
-        active_month: site_view.counts.users_active_month,
-      },
-      local_posts: site_view.counts.posts,
-      local_comments: site_view.counts.comments,
-    },
+    usage: Some(NodeInfoUsage {
+      users: Some(NodeInfoUsers {
+        total: Some(site_view.counts.users),
+        active_halfyear: Some(site_view.counts.users_active_half_year),
+        active_month: Some(site_view.counts.users_active_month),
+      }),
+      local_posts: Some(site_view.counts.posts),
+      local_comments: Some(site_view.counts.comments),
+    }),
     open_registrations,
   };
 
@@ -72,34 +72,35 @@ struct NodeInfoWellKnownLinks {
   pub href: Url,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfo {
-  pub version: String,
-  pub software: NodeInfoSoftware,
-  pub protocols: Vec<String>,
-  pub usage: NodeInfoUsage,
-  pub open_registrations: bool,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfo {
+  pub version: Option<String>,
+  pub software: Option<NodeInfoSoftware>,
+  pub protocols: Option<Vec<String>>,
+  pub usage: Option<NodeInfoUsage>,
+  pub open_registrations: Option<bool>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-struct NodeInfoSoftware {
-  pub name: String,
-  pub version: String,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(default)]
+pub struct NodeInfoSoftware {
+  pub name: Option<String>,
+  pub version: Option<String>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfoUsage {
-  pub users: NodeInfoUsers,
-  pub local_posts: i64,
-  pub local_comments: i64,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfoUsage {
+  pub users: Option<NodeInfoUsers>,
+  pub local_posts: Option<i64>,
+  pub local_comments: Option<i64>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfoUsers {
-  pub total: i64,
-  pub active_halfyear: i64,
-  pub active_month: i64,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfoUsers {
+  pub total: Option<i64>,
+  pub active_halfyear: Option<i64>,
+  pub active_month: Option<i64>,
 }

migrations/2023-02-13-221303_add_instance_software_and_version/down.sql

@@ -0,0 +1,2 @@
+alter table instance drop column software;
+alter table instance drop column version;

migrations/2023-02-13-221303_add_instance_software_and_version/up.sql

@@ -0,0 +1,4 @@
+-- Add Software and Version columns from nodeinfo to the instance table
+
+alter table instance add column software varchar(255);
+alter table instance add column version varchar(255);

scripts/fix-clippy.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -e
 
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+cd $CWD/../
+
 cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \
   -D warnings -D deprecated -D clippy::perf -D clippy::complexity \
   -D clippy::style -D clippy::correctness -D clippy::suspicious \
@@ -10,3 +14,5 @@ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-target
   -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
   -D clippy::unused_self \
   -A clippy::uninlined_format_args
+
+cargo +nightly fmt

src/lib.rs

@@ -42,7 +42,7 @@ use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry}
 use url::Url;
 
 /// Max timeout for http requests
-const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
+pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
 
 /// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
 pub async fn start_lemmy_server() -> Result<(), LemmyError> {
@@ -73,11 +73,6 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
   let pool = build_db_pool(&settings).await?;
   run_advanced_migrations(&pool, &settings).await?;
 
-  // Schedules various cleanup tasks for the DB
-  thread::spawn(move || {
-    scheduled_tasks::setup(db_url).expect("Couldn't set up scheduled_tasks");
-  });
-
   // Initialize the secrets
   let secret = Secret::init(&pool)
     .await
@@ -106,8 +101,9 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
     settings.bind, settings.port
   );
 
+  let user_agent = build_user_agent(&settings);
   let reqwest_client = Client::builder()
-    .user_agent(build_user_agent(&settings))
+    .user_agent(user_agent.clone())
     .timeout(REQWEST_TIMEOUT)
     .build()?;
 
@@ -128,6 +124,11 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
     .with(TracingMiddleware::default())
     .build();
 
+  // Schedules various cleanup tasks for the DB
+  thread::spawn(move || {
+    scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks");
+  });
+
   let chat_server = Arc::new(ChatServer::startup());
 
   // Create Http server with websocket support

src/scheduled_tasks.rs

@@ -2,17 +2,25 @@ use clokwerk::{Scheduler, TimeUnits};
 // Import week days and WeekDay
 use diesel::{sql_query, PgConnection, RunQueryDsl};
 use diesel::{Connection, ExpressionMethods, QueryDsl};
-use lemmy_utils::error::LemmyError;
+use lemmy_db_schema::{
+  source::instance::{Instance, InstanceForm},
+  utils::naive_now,
+};
+use lemmy_routes::nodeinfo::NodeInfo;
+use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT};
+use reqwest::blocking::Client;
 use std::{thread, time::Duration};
 use tracing::info;
 
 /// Schedules various cleanup tasks for lemmy in a background thread
-pub fn setup(db_url: String) -> Result<(), LemmyError> {
+pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> {
   // Setup the connections
   let mut scheduler = Scheduler::new();
 
   let mut conn = PgConnection::establish(&db_url).expect("could not establish connection");
 
+  let mut conn_2 = PgConnection::establish(&db_url).expect("could not establish connection");
+
   active_counts(&mut conn);
   update_banned_when_expired(&mut conn);
 
@@ -33,6 +41,11 @@ pub fn setup(db_url: String) -> Result<(), LemmyError> {
     clear_old_activities(&mut conn);
   });
 
+  update_instance_software(&mut conn_2, &user_agent);
+  scheduler.every(1.days()).run(move || {
+    update_instance_software(&mut conn_2, &user_agent);
+  });
+
   // Manually run the scheduler in an event loop
   loop {
     scheduler.run_pending();
@@ -120,3 +133,67 @@ fn drop_ccnew_indexes(conn: &mut PgConnection) {
     .execute(conn)
     .expect("drop ccnew indexes");
 }
+
+/// Updates the instance software and version
+fn update_instance_software(conn: &mut PgConnection, user_agent: &str) {
+  use lemmy_db_schema::schema::instance;
+  info!("Updating instances software and versions...");
+
+  let client = Client::builder()
+    .user_agent(user_agent)
+    .timeout(REQWEST_TIMEOUT)
+    .build()
+    .expect("couldnt build reqwest client");
+
+  let instances = instance::table
+    .get_results::<Instance>(conn)
+    .expect("no instances found");
+
+  for instance in instances {
+    let node_info_url = format!("https://{}/nodeinfo/2.0.json", instance.domain);
+
+    // Skip it if it can't connect
+    let res = client
+      .get(&node_info_url)
+      .send()
+      .ok()
+      .and_then(|t| t.json::<NodeInfo>().ok());
+
+    if let Some(node_info) = res {
+      let software = node_info.software.as_ref();
+      let form = InstanceForm::builder()
+        .domain(instance.domain)
+        .software(software.and_then(|s| s.name.clone()))
+        .version(software.and_then(|s| s.version.clone()))
+        .updated(Some(naive_now()))
+        .build();
+
+      diesel::update(instance::table.find(instance.id))
+        .set(form)
+        .execute(conn)
+        .expect("update site instance software");
+    }
+  }
+  info!("Done.");
+}
+
+#[cfg(test)]
+mod tests {
+  use lemmy_routes::nodeinfo::NodeInfo;
+  use reqwest::Client;
+
+  #[tokio::test]
+  async fn test_nodeinfo() {
+    let client = Client::builder().build().unwrap();
+    let lemmy_ml_nodeinfo = client
+      .get("https://lemmy.ml/nodeinfo/2.0.json")
+      .send()
+      .await
+      .unwrap()
+      .json::<NodeInfo>()
+      .await
+      .unwrap();
+
+    assert_eq!(lemmy_ml_nodeinfo.software.unwrap().name.unwrap(), "lemmy");
+  }
+}