From 86169871cb51f0524300878b8c9195eff33a91fb Mon Sep 17 00:00:00 2001 From: Clement Tsang <34804052+ClementTsang@users.noreply.github.com> Date: Mon, 30 Nov 2020 19:40:17 -0500 Subject: [PATCH] refactor: Simplify data harvesting (#335) Refactors the data harvesting system function names. --- .vscode/settings.json | 1 + Cargo.lock | 240 +++++++++++++++++++++++++++- Cargo.toml | 2 +- src/app/data_harvester.rs | 38 ++--- src/app/data_harvester/disks.rs | 10 +- src/app/data_harvester/mem.rs | 14 +- src/app/data_harvester/network.rs | 4 +- src/app/data_harvester/processes.rs | 4 +- 8 files changed, 266 insertions(+), 47 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5decc229..8d157fab 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -92,6 +92,7 @@ "rustflags", "rustfmt", "shilangyu", + "smol", "softirq", "splitn", "statm", diff --git a/Cargo.lock b/Cargo.lock index d24e247f..f1522e9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,111 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-channel" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb877970c7b440ead138f6321a3b5395d6061183af779340b65e20c0fede9146" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "vec-arena", +] + +[[package]] +name = "async-fs" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3ca4f8ff117c37c278a2f7415ce9be55560b846b5bc4412aaa5d29c1c3dae2" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458c8f66c246624e7cf87c01451f3392ab77d66a0f105a49d9353b30ea97ced8" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "nb-connect", + "once_cell", + "parking", + "polling", + "vec-arena", + "waker-fn", + "winapi", +] + +[[package]] +name = "async-lock" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1996609732bde4a9988bc42125f55f2af5f3c36370e27c778d5191a4a1b63bfb" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-net" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06de475c85affe184648202401d7622afb32f0f74e02192857d0201a16defbe5" +dependencies = [ + "async-io", + "blocking", + "fastrand", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8cea09c1fb10a317d1b5af8024eeba256d6554763e85ecd90ff8df31c7bbda" +dependencies = [ + "async-io", + "blocking", + "cfg-if 0.1.10", + "event-listener", + "futures-lite", + "once_cell", + "signal-hook", + "winapi", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "atty" version = "0.2.14" @@ -106,6 +211,20 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + [[package]] name = "bottom" version = "0.5.3" @@ -115,7 +234,6 @@ dependencies = [ "backtrace", "battery", "cargo-husky", - "cfg-if 1.0.0", "chrono", "clap", "crossterm", @@ -132,6 +250,7 @@ dependencies = [ "predicates", "regex", "serde", + "smol", "sysinfo", "thiserror", "toml", @@ -142,6 +261,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "cargo-husky" version = "1.5.0" @@ -209,6 +334,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "const_fn" version = "0.4.3" @@ -372,6 +506,21 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fastrand" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +dependencies = [ + "instant", +] + [[package]] name = "fern" version = "0.6.0" @@ -438,6 +587,21 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +[[package]] +name = "futures-lite" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6c079abfac3ab269e2927ec048dabc89d009ebfdda6b8ee86624f30c689658" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.8" @@ -824,6 +988,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "nb-connect" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "nix" version = "0.17.0" @@ -937,6 +1111,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.0" @@ -983,6 +1163,12 @@ dependencies = [ "syn", ] +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + [[package]] name = "pin-utils" version = "0.1.0" @@ -995,6 +1181,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" +[[package]] +name = "polling" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "log", + "wepoll-sys", + "winapi", +] + [[package]] name = "predicates" version = "1.0.5" @@ -1212,6 +1411,24 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +[[package]] +name = "smol" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cf3b5351f3e783c1d79ab5fc604eeed8b8ae9abd36b166e8b87a089efd85e4" +dependencies = [ + "async-channel", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-net", + "async-process", + "blocking", + "futures-lite", + "once_cell", +] + [[package]] name = "socket2" version = "0.3.15" @@ -1388,6 +1605,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "vec-arena" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" + [[package]] name = "vec_map" version = "0.8.2" @@ -1409,6 +1632,12 @@ dependencies = [ "libc", ] +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -1421,6 +1650,15 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wepoll-sys" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +dependencies = [ + "cc", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index 519a1e57..62e72140 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ codegen-units = 1 anyhow = "1.0.34" backtrace = "0.3" battery = "0.7.8" -cfg-if = "1.0.0" chrono = "0.4.19" crossterm = "0.18.2" ctrlc = {version = "3.1", features = ["termination"]} @@ -40,6 +39,7 @@ libc = "0.2" once_cell = "1.5.2" regex = "1.4.2" serde = {version = "1.0", features = ["derive"] } +smol = "1.2.5" sysinfo = "0.15.3" thiserror = "1.0.22" toml = "0.5.7" diff --git a/src/app/data_harvester.rs b/src/app/data_harvester.rs index 2e46559a..0f59b54f 100644 --- a/src/app/data_harvester.rs +++ b/src/app/data_harvester.rs @@ -236,13 +236,10 @@ impl DataCollector { } if self.widgets_to_harvest.use_proc { - // Processes. This is the longest part of the harvesting process... changing this might be - // good in the future. What was tried already: - // * Splitting the internal part into multiple scoped threads (dropped by ~.01 seconds, but upped usage) - if let Ok(process_list) = if cfg!(target_os = "linux") { + if let Ok(process_list) = { #[cfg(target_os = "linux")] { - processes::linux_processes( + processes::get_process_data( &mut self.prev_idle, &mut self.prev_non_idle, &mut self.pid_mapping, @@ -256,21 +253,12 @@ impl DataCollector { } #[cfg(not(target_os = "linux"))] { - Ok(Vec::new()) - } - } else { - #[cfg(not(target_os = "linux"))] - { - processes::windows_macos_processes( + processes::get_process_data( &self.sys, self.use_current_cpu_total, self.mem_total_kb, ) } - #[cfg(target_os = "linux")] - { - Ok(Vec::new()) - } } { self.data.list_of_processes = Some(process_list); } @@ -288,7 +276,7 @@ impl DataCollector { let network_data_fut = { #[cfg(any(target_os = "windows", target_arch = "aarch64", target_arch = "arm"))] { - network::arm_or_windows_network_data( + network::get_network_data( &self.sys, self.last_collection_time, &mut self.total_rx, @@ -299,7 +287,7 @@ impl DataCollector { } #[cfg(not(any(target_os = "windows", target_arch = "aarch64", target_arch = "arm")))] { - network::non_arm_or_windows_network_data( + network::get_network_data( self.last_collection_time, &mut self.total_rx, &mut self.total_tx, @@ -311,45 +299,45 @@ impl DataCollector { let mem_data_fut = { #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] { - mem::arm_mem_data(&self.sys, self.widgets_to_harvest.use_mem) + mem::get_mem_data(&self.sys, self.widgets_to_harvest.use_mem) } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] { - mem::non_arm_mem_data(self.widgets_to_harvest.use_mem) + mem::get_mem_data(self.widgets_to_harvest.use_mem) } }; let swap_data_fut = { #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] { - mem::arm_swap_data(&self.sys, self.widgets_to_harvest.use_mem) + mem::get_swap_data(&self.sys, self.widgets_to_harvest.use_mem) } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] { - mem::non_arm_swap_data(self.widgets_to_harvest.use_mem) + mem::get_swap_data(self.widgets_to_harvest.use_mem) } }; let disk_data_fut = { #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] { - disks::arm_disk_usage(&self.sys, self.widgets_to_harvest.use_disk) + disks::get_disk_usage(&self.sys, self.widgets_to_harvest.use_disk) } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] { - disks::non_arm_disk_usage(self.widgets_to_harvest.use_disk) + disks::get_disk_usage(self.widgets_to_harvest.use_disk) } }; let disk_io_usage_fut = { #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] { - disks::arm_io_usage(&self.sys, self.widgets_to_harvest.use_disk) + disks::get_io_usage(&self.sys, self.widgets_to_harvest.use_disk) } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] { - disks::non_arm_io_usage(false, self.widgets_to_harvest.use_disk) + disks::get_io_usage(false, self.widgets_to_harvest.use_disk) } }; let temp_data_fut = { diff --git a/src/app/data_harvester/disks.rs b/src/app/data_harvester/disks.rs index 59e9d13c..cb7ca0f4 100644 --- a/src/app/data_harvester/disks.rs +++ b/src/app/data_harvester/disks.rs @@ -18,9 +18,8 @@ pub struct IOData { pub type IOHarvest = std::collections::HashMap>; -/// Meant for ARM use. #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] -pub async fn arm_io_usage( +pub async fn get_io_usage( _sys: &sysinfo::System, _actually_get: bool, ) -> crate::utils::error::Result> { let io_hash: std::collections::HashMap> = @@ -32,9 +31,8 @@ pub async fn arm_io_usage( // work. See https://github.com/GuillaumeGomez/sysinfo/issues/304. } -/// Meant for ARM use. #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] -pub async fn arm_disk_usage( +pub async fn get_disk_usage( sys: &sysinfo::System, actually_get: bool, ) -> crate::utils::error::Result>> { use sysinfo::{DiskExt, SystemExt}; @@ -61,7 +59,7 @@ pub async fn arm_disk_usage( } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] -pub async fn non_arm_io_usage( +pub async fn get_io_usage( get_physical: bool, actually_get: bool, ) -> crate::utils::error::Result> { if !actually_get { @@ -104,7 +102,7 @@ pub async fn non_arm_io_usage( } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] -pub async fn non_arm_disk_usage( +pub async fn get_disk_usage( actually_get: bool, ) -> crate::utils::error::Result>> { if !actually_get { diff --git a/src/app/data_harvester/mem.rs b/src/app/data_harvester/mem.rs index e108e867..5f91b40e 100644 --- a/src/app/data_harvester/mem.rs +++ b/src/app/data_harvester/mem.rs @@ -13,9 +13,8 @@ impl Default for MemHarvest { } } -/// Meant for ARM use. #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] -pub async fn arm_mem_data( +pub async fn get_mem_data( sys: &sysinfo::System, actually_get: bool, ) -> crate::utils::error::Result> { use sysinfo::SystemExt; @@ -29,9 +28,8 @@ pub async fn arm_mem_data( })) } -/// Meant for ARM use. #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] -pub async fn arm_swap_data( +pub async fn get_swap_data( sys: &sysinfo::System, actually_get: bool, ) -> crate::utils::error::Result> { use sysinfo::SystemExt; @@ -46,9 +44,7 @@ pub async fn arm_swap_data( } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] -pub async fn non_arm_mem_data( - actually_get: bool, -) -> crate::utils::error::Result> { +pub async fn get_mem_data(actually_get: bool) -> crate::utils::error::Result> { if !actually_get { return Ok(None); } @@ -65,9 +61,7 @@ pub async fn non_arm_mem_data( } #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] -pub async fn non_arm_swap_data( - actually_get: bool, -) -> crate::utils::error::Result> { +pub async fn get_swap_data(actually_get: bool) -> crate::utils::error::Result> { if !actually_get { return Ok(None); } diff --git a/src/app/data_harvester/network.rs b/src/app/data_harvester/network.rs index 95e15476..9b37fc7a 100644 --- a/src/app/data_harvester/network.rs +++ b/src/app/data_harvester/network.rs @@ -17,7 +17,7 @@ impl NetworkHarvest { /// Meant for Windows and ARM use. #[cfg(any(target_os = "windows", target_arch = "aarch64", target_arch = "arm"))] -pub async fn arm_or_windows_network_data( +pub async fn get_network_data( sys: &sysinfo::System, prev_net_access_time: Instant, prev_net_rx: &mut u64, prev_net_tx: &mut u64, curr_time: Instant, actually_get: bool, ) -> Option { @@ -58,7 +58,7 @@ pub async fn arm_or_windows_network_data( } #[cfg(not(any(target_os = "windows", target_arch = "aarch64", target_arch = "arm")))] -pub async fn non_arm_or_windows_network_data( +pub async fn get_network_data( prev_net_access_time: Instant, prev_net_rx: &mut u64, prev_net_tx: &mut u64, curr_time: Instant, actually_get: bool, ) -> Option { diff --git a/src/app/data_harvester/processes.rs b/src/app/data_harvester/processes.rs index a182db23..a79f8585 100644 --- a/src/app/data_harvester/processes.rs +++ b/src/app/data_harvester/processes.rs @@ -369,7 +369,7 @@ fn read_proc( } #[cfg(target_os = "linux")] -pub fn linux_processes( +pub fn get_process_data( prev_idle: &mut f64, prev_non_idle: &mut f64, pid_mapping: &mut HashMap, use_current_cpu_total: bool, time_difference_in_secs: u64, mem_total_kb: u64, page_file_kb: u64, @@ -409,7 +409,7 @@ pub fn linux_processes( } #[cfg(not(target_os = "linux"))] -pub fn windows_macos_processes( +pub fn get_process_data( sys: &System, use_current_cpu_total: bool, mem_total_kb: u64, ) -> crate::utils::error::Result> { let mut process_vector: Vec = Vec::new();