Add console-subscriber (#2003)

* Add console-subscriber

* Add larg event buffer capacity

* Add console port to lemmy prod container

* Expose pict-rs console port

* Update pictrs
This commit is contained in:
Riley 2022-01-07 08:53:45 -06:00 committed by GitHub
parent 3032a2745b
commit 41b90bb162
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 289 additions and 32 deletions

2
.cargo/config Normal file
View file

@ -0,0 +1,2 @@
[build]
rustflags = ["--cfg", "tokio_unstable"]

265
Cargo.lock generated
View file

@ -690,6 +690,40 @@ dependencies = [
"xdg",
]
[[package]]
name = "console-api"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "033fddce299c93dd44ae21d5f5a6e749baa5d103784bcdde65701c07272a9fde"
dependencies = [
"prost 0.9.0",
"prost-types 0.9.0",
"tonic 0.6.2",
"tonic-build 0.6.2",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2380cc150266375aeda8f9aeadc5527395c1a8807ecf9fa97a46d1bb760ec5b"
dependencies = [
"console-api",
"futures",
"hdrhistogram",
"humantime",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic 0.6.2",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "convert_case"
version = "0.4.0"
@ -1058,6 +1092,70 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encoding"
version = "0.2.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
dependencies = [
"encoding-index-japanese",
"encoding-index-korean",
"encoding-index-simpchinese",
"encoding-index-singlebyte",
"encoding-index-tradchinese",
]
[[package]]
name = "encoding-index-japanese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-korean"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-simpchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-singlebyte"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-tradchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding_index_tests"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]]
name = "encoding_rs"
version = "0.8.30"
@ -1115,6 +1213,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
[[package]]
name = "fixedbitset"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
[[package]]
name = "flate2"
version = "1.0.22"
@ -1348,6 +1452,19 @@ dependencies = [
"ahash",
]
[[package]]
name = "hdrhistogram"
version = "7.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6490be71f07a5f62b564bc58e36953f675833df11c7e4a0647bee7a07ca1ec5e"
dependencies = [
"base64 0.13.0",
"byteorder",
"flate2",
"nom 7.1.0",
"num-traits",
]
[[package]]
name = "heck"
version = "0.3.3"
@ -1494,6 +1611,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.16"
@ -1971,6 +2094,7 @@ dependencies = [
"cargo-husky",
"chrono",
"clokwerk",
"console-subscriber",
"diesel",
"diesel_migrations",
"doku",
@ -2018,6 +2142,7 @@ dependencies = [
"deser-hjson",
"diesel",
"doku",
"encoding",
"futures",
"http",
"itertools",
@ -2542,11 +2667,11 @@ dependencies = [
"futures",
"http",
"opentelemetry",
"prost",
"prost 0.8.0",
"thiserror",
"tokio",
"tonic",
"tonic-build",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -2646,7 +2771,17 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
dependencies = [
"fixedbitset",
"fixedbitset 0.2.0",
"indexmap",
]
[[package]]
name = "petgraph"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"
dependencies = [
"fixedbitset 0.4.1",
"indexmap",
]
@ -2784,7 +2919,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
dependencies = [
"bytes",
"prost-derive",
"prost-derive 0.8.0",
]
[[package]]
name = "prost"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
dependencies = [
"bytes",
"prost-derive 0.9.0",
]
[[package]]
@ -2798,9 +2943,29 @@ dependencies = [
"itertools",
"log",
"multimap",
"petgraph",
"prost",
"prost-types",
"petgraph 0.5.1",
"prost 0.8.0",
"prost-types 0.8.0",
"tempfile",
"which",
]
[[package]]
name = "prost-build"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
dependencies = [
"bytes",
"heck",
"itertools",
"lazy_static",
"log",
"multimap",
"petgraph 0.6.0",
"prost 0.9.0",
"prost-types 0.9.0",
"regex",
"tempfile",
"which",
]
@ -2818,6 +2983,19 @@ dependencies = [
"syn 1.0.82",
]
[[package]]
name = "prost-derive"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
dependencies = [
"anyhow",
"itertools",
"proc-macro2 1.0.33",
"quote 1.0.10",
"syn 1.0.82",
]
[[package]]
name = "prost-types"
version = "0.8.0"
@ -2825,7 +3003,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
dependencies = [
"bytes",
"prost",
"prost 0.8.0",
]
[[package]]
name = "prost-types"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
dependencies = [
"bytes",
"prost 0.9.0",
]
[[package]]
@ -3693,11 +3881,10 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.14.0"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144"
checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838"
dependencies = [
"autocfg",
"bytes",
"libc",
"memchr",
@ -3708,6 +3895,7 @@ dependencies = [
"pin-project-lite",
"signal-hook-registry",
"tokio-macros",
"tracing",
"winapi",
]
@ -3723,9 +3911,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "1.6.0"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e"
checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
dependencies = [
"proc-macro2 1.0.33",
"quote 1.0.10",
@ -3806,8 +3994,39 @@ dependencies = [
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"prost-derive",
"prost 0.8.0",
"prost-derive 0.8.0",
"tokio",
"tokio-stream",
"tokio-util",
"tower",
"tower-layer",
"tower-service",
"tracing",
"tracing-futures",
]
[[package]]
name = "tonic"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
dependencies = [
"async-stream",
"async-trait",
"base64 0.13.0",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost 0.9.0",
"prost-derive 0.9.0",
"tokio",
"tokio-stream",
"tokio-util",
@ -3825,7 +4044,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08"
dependencies = [
"proc-macro2 1.0.33",
"prost-build",
"prost-build 0.8.0",
"quote 1.0.10",
"syn 1.0.82",
]
[[package]]
name = "tonic-build"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
dependencies = [
"proc-macro2 1.0.33",
"prost-build 0.9.0",
"quote 1.0.10",
"syn 1.0.82",
]

View file

@ -54,11 +54,12 @@ tracing-actix-web = { version = "0.5.0-beta.8", default-features = false }
tracing-error = "0.2.0"
tracing-log = "0.1.2"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
console-subscriber = "0.1.0"
strum = "0.23.0"
url = { version = "2.2.2", features = ["serde"] }
openssl = "0.10.38"
http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
tokio = { version = "1.14.0", features = ["sync"] }
tokio = { version = "1.14.0", features = ["sync", "tracing"] }
anyhow = "1.0.51"
reqwest = { version = "0.11.7", features = ["json"] }
reqwest-middleware = "0.1.3"

View file

@ -39,7 +39,7 @@ http = "0.2.5"
http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
base64 = "0.13.0"
tokio = "1.14.0"
futures = "0.3.18"
futures = "0.3.17"
itertools = "0.10.3"
uuid = { version = "0.8.2", features = ["serde", "v4"] }
sha2 = "0.10.0"

View file

@ -35,7 +35,7 @@ http = "0.2.5"
http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
base64 = "0.13.0"
tokio = "1.14.0"
futures = "0.3.18"
futures = "0.3.17"
itertools = "0.10.3"
uuid = { version = "0.8.2", features = ["serde", "v4"] }
sha2 = "0.10.0"

View file

@ -39,7 +39,7 @@ percent-encoding = "2.1.0"
http = "0.2.5"
http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["server", "sha-2"] }
tokio = "1.14.0"
futures = "0.3.18"
futures = "0.3.17"
itertools = "0.10.3"
uuid = { version = "0.8.2", features = ["serde", "v4"] }
sha2 = "0.10.0"

View file

@ -82,9 +82,10 @@ async fn do_send(task: SendActivityTask, client: &ClientWithMiddleware) -> Resul
if !o.status().is_success() {
let status = o.status();
let text = o.text().await?;
warn!(
"Send {} to {} failed with status {}: {}",
task.activity_id, task.inbox, status, text
task.activity_id, task.inbox, status, text,
);
}
}

View file

@ -26,7 +26,7 @@ actix-http = "3.0.0-beta.15"
sha2 = "0.10.0"
anyhow = "1.0.51"
chrono = { version = "0.4.19", features = ["serde"] }
futures = "0.3.18"
futures = "0.3.17"
reqwest = { version = "0.11.7", features = ["stream"] }
reqwest-middleware = "0.1.3"
rss = "2.0.0"

View file

@ -36,7 +36,7 @@ reqwest-middleware = "0.1.3"
tokio = { version = "1.14.0", features = ["sync"] }
strum = "0.23.0"
strum_macros = "0.23.1"
futures = "0.3.18"
futures = "0.3.17"
diesel = "1.4.8"
http = "0.2.5"
deser-hjson = "1.0.2"

View file

@ -7,6 +7,7 @@ WORKDIR /app
# Cargo chef plan
FROM chef as planner
ENV RUSTFLAGS="--cfg tokio_unstable"
# Copy dirs
COPY . .
@ -16,6 +17,7 @@ RUN cargo chef prepare --recipe-path recipe.json
FROM chef as builder
ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
ARG RUSTRELEASEDIR="debug"
ENV RUSTFLAGS="--cfg tokio_unstable"
COPY --from=planner /app/recipe.json ./recipe.json
RUN cargo chef cook --recipe-path recipe.json --target ${CARGO_BUILD_TARGET}

View file

@ -16,6 +16,7 @@ services:
image: lemmy-dev:latest
ports:
- "8536:8536"
- "6669:6669"
restart: always
environment:
- RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
@ -50,10 +51,12 @@ services:
restart: always
pictrs:
image: asonix/pictrs:0.3.0-beta.11
image: asonix/pictrs:0.3.0-beta.12-r1
user: 991:991
environment:
- PICTRS_OPENTELEMETRY_URL=http://otel:4137
ports:
- "6670:6669"
volumes:
- ./volumes/pictrs:/mnt
restart: always

View file

@ -15,6 +15,7 @@ services:
image: dessalines/lemmy:0.14.3
ports:
- "127.0.0.1:8536:8536"
- "127.0.0.1:6669:6669"
restart: always
environment:
- RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
@ -37,9 +38,10 @@ services:
- lemmy
pictrs:
image: asonix/pictrs:0.3.0-beta.11
image: asonix/pictrs:0.3.0-beta.12-r1
ports:
- "127.0.0.1:8537:8080"
- "127.0.0.1:6670:6669"
user: 991:991
volumes:
- ./volumes/pictrs:/mnt

View file

@ -4,6 +4,7 @@ pub mod code_migrations;
pub mod root_span_builder;
pub mod scheduled_tasks;
use console_subscriber::ConsoleLayer;
use lemmy_utils::LemmyError;
use opentelemetry::{
sdk::{propagation::TraceContextPropagator, Resource},
@ -13,20 +14,32 @@ use opentelemetry_otlp::WithExportConfig;
use tracing::subscriber::set_global_default;
use tracing_error::ErrorLayer;
use tracing_log::LogTracer;
use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry};
pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> {
LogTracer::init()?;
opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let format_layer = tracing_subscriber::fmt::layer();
let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into());
let targets = log_description
.trim()
.trim_matches('"')
.parse::<Targets>()?;
let format_layer = tracing_subscriber::fmt::layer().with_filter(targets.clone());
let console_layer = ConsoleLayer::builder()
.with_default_env()
.server_addr(([0, 0, 0, 0], 6669))
.event_buffer_capacity(1024 * 1024)
.spawn();
let subscriber = Registry::default()
.with(env_filter)
.with(format_layer)
.with(ErrorLayer::default());
.with(ErrorLayer::default())
.with(console_layer);
if let Some(url) = opentelemetry_url {
let tracer = opentelemetry_otlp::new_pipeline()
@ -42,7 +55,9 @@ pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> {
)
.install_batch(opentelemetry::runtime::Tokio)?;
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
let otel_layer = tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(targets);
let subscriber = subscriber.with(otel_layer);