docs: use rust libraries for downloading and unzipping tldr

This commit is contained in:
Terts Diepraam 2022-02-17 22:24:41 +01:00
parent f57e3470ae
commit 69a94e412b
6 changed files with 364 additions and 47 deletions

View file

@ -44,6 +44,7 @@ termsize
termwidth
textwrap
thiserror
ureq
walkdir
winapi
xattr

302
Cargo.lock generated
View file

@ -8,6 +8,12 @@ version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.4.7"
@ -73,6 +79,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bigdecimal"
version = "0.3.0"
@ -167,6 +179,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "bumpalo"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "byte-unit"
version = "4.0.13"
@ -228,6 +246,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "chunked_transfer"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e"
[[package]]
name = "clang-sys"
version = "1.3.0"
@ -329,6 +353,7 @@ dependencies = [
"time",
"unindent",
"unix_socket",
"ureq",
"users",
"uu_arch",
"uu_base32",
@ -432,6 +457,7 @@ dependencies = [
"uu_yes",
"uucore",
"walkdir",
"zip",
]
[[package]]
@ -546,6 +572,15 @@ dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.2"
@ -792,12 +827,34 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "flate2"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
dependencies = [
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
dependencies = [
"matches",
"percent-encoding",
]
[[package]]
name = "fs_extra"
version = "1.2.0"
@ -927,6 +984,17 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "idna"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
dependencies = [
"matches",
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "if_rust_version"
version = "1.0.0"
@ -967,6 +1035,15 @@ dependencies = [
"either",
]
[[package]]
name = "js-sys"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "keccak"
version = "0.1.0"
@ -1044,6 +1121,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
[[package]]
name = "matches"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "md5"
version = "0.3.8"
@ -1089,6 +1172,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "mio"
version = "0.7.14"
@ -1375,6 +1468,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "percent-encoding"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "phf"
version = "0.10.1"
@ -1655,6 +1754,21 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1"
[[package]]
name = "ring"
version = "0.16.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
dependencies = [
"cc",
"libc",
"once_cell",
"spin",
"untrusted",
"web-sys",
"winapi 0.3.9",
]
[[package]]
name = "rlimit"
version = "0.4.0"
@ -1681,6 +1795,18 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustls"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700"
dependencies = [
"log",
"ring",
"sct",
"webpki",
]
[[package]]
name = "same-file"
version = "1.0.6"
@ -1696,6 +1822,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "selinux"
version = "0.2.5"
@ -1838,6 +1974,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@ -2009,6 +2151,21 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "tinyvec"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "toml"
version = "0.5.8"
@ -2024,6 +2181,12 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicode-bidi"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
[[package]]
name = "unicode-linebreak"
version = "0.1.2"
@ -2033,6 +2196,15 @@ dependencies = [
"regex",
]
[[package]]
name = "unicode-normalization"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.8.0"
@ -2073,6 +2245,41 @@ dependencies = [
"libc",
]
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "ureq"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5"
dependencies = [
"base64",
"chunked_transfer",
"flate2",
"log",
"once_cell",
"rustls",
"url",
"webpki",
"webpki-roots",
]
[[package]]
name = "url"
version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
dependencies = [
"form_urlencoded",
"idna",
"matches",
"percent-encoding",
]
[[package]]
name = "users"
version = "0.10.0"
@ -3171,6 +3378,89 @@ version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
dependencies = [
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote 1.0.14",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
dependencies = [
"quote 1.0.14",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
dependencies = [
"proc-macro2",
"quote 1.0.14",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
[[package]]
name = "web-sys"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449"
dependencies = [
"webpki",
]
[[package]]
name = "which"
version = "4.2.2"
@ -3248,3 +3538,15 @@ name = "z85"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af896e93db81340b74b65f74276a99b210c086f3d34ed0abf433182a462af856"
[[package]]
name = "zip"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815"
dependencies = [
"byteorder",
"crc32fast",
"flate2",
"thiserror",
]

View file

@ -252,6 +252,8 @@ lazy_static = { version="1.3" }
textwrap = { version="0.14", features=["terminal_size"] }
uucore = { version=">=0.0.11", package="uucore", path="src/uucore" }
selinux = { version="0.2", optional = true }
ureq = "2.4.0"
zip = { version = "0.5.13", default_features=false, features=["deflate"] }
# * uutils
uu_test = { optional=true, version="0.0.12", package="uu_test", path="src/uu/test" }
#

1
docs/.gitignore vendored
View file

@ -1,4 +1,3 @@
book
src/utils
src/SUMMARY.md
tldr/

View file

@ -1,6 +1,4 @@
# spell-checker:ignore tldr
clean:
rm -rf book
rm -f src/SUMMARY.md
rm -f src/utils/*
rm -rf tldr

View file

@ -7,27 +7,21 @@
use clap::App;
use std::ffi::OsString;
use std::fs::File;
use std::io::{self, Read, Write};
use std::process::Command;
use std::io::Cursor;
use std::io::{self, Read, Seek, Write};
use zip::ZipArchive;
include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));
fn main() -> io::Result<()> {
let _ = std::fs::create_dir("docs/tldr");
println!("Downloading tldr archive");
Command::new("curl")
.arg("https://tldr.sh/assets/tldr.zip")
.arg("--output")
.arg("docs/tldr/tldr.zip")
.output()?;
println!("Unzipping tldr archive");
Command::new("unzip")
.arg("-o")
.arg("docs/tldr/tldr.zip")
.arg("-d")
.arg("docs/tldr")
.output()?;
let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip")
.call()
.unwrap()
.into_reader();
let mut buffer = Vec::new();
zip_reader.read_to_end(&mut buffer).unwrap();
let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap();
let utils = util_map::<Box<dyn Iterator<Item = OsString>>>();
match std::fs::create_dir("docs/src/utils/") {
@ -58,7 +52,7 @@ fn main() -> io::Result<()> {
}
let p = format!("docs/src/utils/{}.md", name);
if let Ok(f) = File::create(&p) {
write_markdown(f, &mut app(), name)?;
write_markdown(f, &mut app(), name, &mut tldr_zip)?;
println!("Wrote to '{}'", p);
} else {
println!("Error writing to {}", p);
@ -68,12 +62,17 @@ fn main() -> io::Result<()> {
Ok(())
}
fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<()> {
fn write_markdown(
mut w: impl Write,
app: &mut App,
name: &str,
tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
write!(w, "# {}\n\n", name)?;
write_version(&mut w, app)?;
write_usage(&mut w, app, name)?;
write_description(&mut w, app)?;
write_examples(&mut w, name)?;
write_examples(&mut w, name, tldr_zip)?;
write_options(&mut w, app)
}
@ -101,33 +100,49 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> {
}
}
fn write_examples(w: &mut impl Write, name: &str) -> io::Result<()> {
if let Ok(mut file) = std::fs::File::open(format!("docs/tldr/pages/common/{}.md", name))
.or_else(|_| std::fs::File::open(format!("docs/tldr/pages/linux/{}.md", name)))
{
let mut content = String::new();
file.read_to_string(&mut content)?;
writeln!(w, "## Examples")?;
writeln!(w)?;
for line in content.lines().skip_while(|l| !l.starts_with('-')) {
if let Some(l) = line.strip_prefix("- ") {
writeln!(w, "{}", l)?;
} else if line.starts_with('`') {
writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?;
} else if line.is_empty() {
writeln!(w)?;
} else {
println!("Not sure what to do with this line:");
println!("{}", line);
}
}
writeln!(w)?;
writeln!(w, "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md).")?;
fn write_examples(
w: &mut impl Write,
name: &str,
tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) {
f
} else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) {
f
} else {
println!("No examples found for: {}", name);
return Ok(());
};
writeln!(w, "## Examples")?;
writeln!(w)?;
for line in content.lines().skip_while(|l| !l.starts_with('-')) {
if let Some(l) = line.strip_prefix("- ") {
writeln!(w, "{}", l)?;
} else if line.starts_with('`') {
writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?;
} else if line.is_empty() {
writeln!(w)?;
} else {
println!("Not sure what to do with this line:");
println!("{}", line);
}
}
Ok(())
writeln!(w)?;
writeln!(
w,
"> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)."
)?;
writeln!(w, ">")?;
writeln!(
w,
"> Please note that, as uutils is a work in progress, some examples might fail."
)
}
fn get_zip_content(archive: &mut ZipArchive<impl Read + Seek>, name: &str) -> Option<String> {
let mut s = String::new();
archive.by_name(name).ok()?.read_to_string(&mut s).unwrap();
Some(s)
}
fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> {