diff --git a/Cargo.lock b/Cargo.lock index a9b6b51227..cf0b7954ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,30 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -25,12 +49,38 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backtrace" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -48,6 +98,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "cc" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" + [[package]] name = "cfg-if" version = "1.0.0" @@ -68,13 +124,53 @@ dependencies = [ ] [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "core-foundation-sys" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ - "termcolor", - "unicode-width", + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if", + "lazy_static", ] [[package]] @@ -125,12 +221,39 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541" + [[package]] name = "either" version = "1.6.1" @@ -142,12 +265,16 @@ name = "engine-q" version = "0.1.0" dependencies = [ "assert_cmd", - "codespan-reporting", + "crossterm", + "miette", "nu-cli", "nu-command", "nu-engine", + "nu-json", "nu-parser", + "nu-path", "nu-protocol", + "nu-table", "pretty_assertions", "reedline", "tempfile", @@ -164,6 +291,27 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "instant" version = "0.1.11" @@ -173,6 +321,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "is_ci" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" + [[package]] name = "itertools" version = "0.10.1" @@ -182,6 +336,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "lazy_static" version = "1.4.0" @@ -190,9 +350,25 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" +<<<<<<< HEAD version = "0.2.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" +======= +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +dependencies = [ + "serde", + "serde_test", +] +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 [[package]] name = "lock_api" @@ -218,6 +394,55 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miette" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b98aebb9d23c72cb22c089834ea59be059c6f462e844fd9fd18dd0168ad149c" +dependencies = [ + "atty", + "backtrace", + "miette-derive", + "once_cell", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "term_size", + "textwrap", + "thiserror", +] + +[[package]] +name = "miette-derive" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a7cd3347eb52480d4ba59d71ce2b48b8b128034e17068c7e2ff3766c2e87a3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.13" @@ -264,21 +489,30 @@ dependencies = [ name = "nu-cli" version = "0.1.0" dependencies = [ - "codespan-reporting", + "miette", "nu-ansi-term", "nu-engine", "nu-parser", "nu-protocol", "reedline", + "thiserror", ] [[package]] name = "nu-command" version = "0.1.0" dependencies = [ + "glob", "nu-engine", +<<<<<<< HEAD "nu-parser", +======= + "nu-json", +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 "nu-protocol", + "nu-table", + "sysinfo", + "thiserror", ] [[package]] @@ -286,22 +520,56 @@ name = "nu-engine" version = "0.1.0" dependencies = [ "nu-parser", + "nu-path", "nu-protocol", ] +[[package]] +name = "nu-json" +version = "0.37.1" +dependencies = [ + "lazy_static", + "linked-hash-map", + "nu-path", + "num-traits", + "regex", + "serde", + "serde_json", +] + [[package]] name = "nu-parser" version = "0.1.0" dependencies = [ - "codespan-reporting", + "miette", "nu-protocol", + "thiserror", +] + +[[package]] +name = "nu-path" +version = "0.37.1" +dependencies = [ + "dirs-next", + "dunce", ] [[package]] name = "nu-protocol" version = "0.1.0" dependencies = [ - "codespan-reporting", + "miette", + "serde", + "thiserror", +] + +[[package]] +name = "nu-table" +version = "0.36.0" +dependencies = [ + "nu-ansi-term", + "regex", + "unicode-width", ] [[package]] @@ -323,6 +591,31 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + [[package]] name = "output_vt100" version = "0.1.2" @@ -338,6 +631,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a61765925aec40abdb23812a3a1a01fafc6ffb9da22768b2ce665a9e84e527c" + [[package]] name = "parking_lot" version = "0.11.2" @@ -466,6 +765,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -475,6 +799,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + [[package]] name = "reedline" version = "0.2.0" @@ -488,12 +822,29 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -503,6 +854,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + [[package]] name = "scopeguard" version = "1.1.0" @@ -529,6 +892,26 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_test" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82178225dbdeae2d5d190e8649287db6a3a32c6d24da22ae3146325aa353e4c" +dependencies = [ + "serde", +] + [[package]] name = "signal-hook" version = "0.3.10" @@ -565,6 +948,40 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +[[package]] +name = "smawk" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" + +[[package]] +name = "supports-color" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f5b0f9e689dd52e27228469dd68b7416b60d75b7571ae9060a5f4c50048fee" +dependencies = [ + "atty", + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590b34f7c5f01ecc9d78dba4b3f445f31df750a67621cf31626f3b7441ce6406" +dependencies = [ + "atty", +] + +[[package]] +name = "supports-unicode" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5fa283a620b255940913bd962cda2e6320e3799041f96ac0d7191ff2b4622f" +dependencies = [ + "atty", +] + [[package]] name = "syn" version = "1.0.77" @@ -576,6 +993,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffff4a02fa61eee51f95210fc9c98ea6eeb46bb071adeafd61e1a0b9b22c6a6d" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "tempfile" version = "3.2.0" @@ -591,12 +1023,44 @@ dependencies = [ ] [[package]] -name = "termcolor" -version = "1.1.2" +name = "term_size" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" dependencies = [ - "winapi-util", + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -616,6 +1080,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" +[[package]] +name = "unicode-linebreak" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a52dcaab0c48d931f7cc8ef826fa51690a08e1ea55117ef26f89864f532383f" +dependencies = [ + "regex", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -665,15 +1138,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 225d8c4f8a..826e3d542e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,16 @@ members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser", "crates/nu-c [dependencies] reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } -codespan-reporting = "0.11.1" +crossterm = "0.21.*" nu-cli = { path="./crates/nu-cli" } nu-command = { path="./crates/nu-command" } nu-engine = { path="./crates/nu-engine" } +nu-json = { path="./crates/nu-json" } nu-parser = { path="./crates/nu-parser" } +nu-path = { path="./crates/nu-path" } nu-protocol = { path = "./crates/nu-protocol" } - +nu-table = { path = "./crates/nu-table" } +miette = "3.0.0" # mimalloc = { version = "*", default-features = false } [dev-dependencies] diff --git a/README.md b/README.md index 5c7e39094b..cec4a5df32 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ # Engine-q -Engine-q is a smaller project to reimplement some of the core functionality in Nushell. It's still in an alpha state, and there is still a lot to do: please see TODO.md +Engine-q is an experimental project to replace the core functionality in Nushell (parser, engine, protocol). It's still in an alpha state, and there is still a lot to do: please see TODO.md + +If you'd like to help out, come join us on the [discord](https://discord.gg/NtAbbGn) or propose some work in an issue or PR draft. We're currently looking to begin porting Nushell commands to engine-q. diff --git a/TODO.md b/TODO.md index 6ff791e61c..fdbf9c3020 100644 --- a/TODO.md +++ b/TODO.md @@ -17,16 +17,24 @@ - [x] Column path - [x] ...rest without calling it rest - [x] Iteration (`each`) over tables +- [x] Row conditions +- [x] Simple completions +- [x] Detecting `$it` currently only looks at top scope but should find any free `$it` in the expression (including subexprs) +- [x] Signature needs to make parameters visible in scope before block is parsed +- [x] Externals +- [x] Modules and imports +- [ ] Exports +- [ ] Support for `$in` +- [ ] Value serialization - [ ] Handling rows with missing columns during a cell path - [ ] Error shortcircuit (stopping on first error) - [ ] ctrl-c support - [ ] operator overflow - [ ] finish operator type-checking - [ ] Source -- [ ] Autoenv -- [ ] Externals -- [ ] let [first, rest] = [1, 2, 3] (design question: how do you pattern match a table?) +- [ ] Overlays (replacement for `autoenv`) ## Maybe: - [ ] default param values? - [ ] Unary not? +- [ ] let [first, rest] = [1, 2, 3] (design question: how do you pattern match a table?) diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index a00e026d48..117fe77d98 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" nu-engine = { path = "../nu-engine" } nu-parser = { path = "../nu-parser" } nu-protocol = { path = "../nu-protocol" } -codespan-reporting = "0.11.1" +miette = { version = "3.0.0", features = ["fancy"] } +thiserror = "1.0.29" nu-ansi-term = "0.36.0" reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } diff --git a/crates/nu-cli/src/completions.rs b/crates/nu-cli/src/completions.rs new file mode 100644 index 0000000000..2140ca0da9 --- /dev/null +++ b/crates/nu-cli/src/completions.rs @@ -0,0 +1,90 @@ +use std::{cell::RefCell, rc::Rc}; + +use nu_engine::eval_block; +use nu_parser::{flatten_block, parse}; +use nu_protocol::{ + engine::{EngineState, EvaluationContext, Stack, StateWorkingSet}, + Value, +}; +use reedline::Completer; + +pub struct NuCompleter { + engine_state: Rc>, +} + +impl NuCompleter { + pub fn new(engine_state: Rc>) -> Self { + Self { engine_state } + } +} + +impl Completer for NuCompleter { + fn complete(&self, line: &str, pos: usize) -> Vec<(reedline::Span, String)> { + let engine_state = self.engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + let offset = working_set.next_span_start(); + let pos = offset + pos; + let (output, _err) = parse(&mut working_set, Some("completer"), line.as_bytes(), false); + + let flattened = flatten_block(&working_set, &output); + + for flat in flattened { + if pos >= flat.0.start && pos <= flat.0.end { + match &flat.1 { + nu_parser::FlatShape::Custom(custom_completion) => { + let prefix = working_set.get_span_contents(flat.0).to_vec(); + + let (block, ..) = + parse(&mut working_set, None, custom_completion.as_bytes(), false); + let context = EvaluationContext { + engine_state: self.engine_state.clone(), + stack: Stack::default(), + }; + let result = eval_block(&context, &block, Value::nothing()); + + let v: Vec<_> = match result { + Ok(Value::List { vals, .. }) => vals + .into_iter() + .map(move |x| { + let s = x.as_string().expect("FIXME"); + + ( + reedline::Span { + start: flat.0.start - offset, + end: flat.0.end - offset, + }, + s, + ) + }) + .filter(|x| x.1.as_bytes().starts_with(&prefix)) + .collect(), + _ => vec![], + }; + + return v; + } + nu_parser::FlatShape::External | nu_parser::FlatShape::InternalCall => { + let prefix = working_set.get_span_contents(flat.0); + let results = working_set.find_commands_by_prefix(prefix); + + return results + .into_iter() + .map(move |x| { + ( + reedline::Span { + start: flat.0.start - offset, + end: flat.0.end - offset, + }, + String::from_utf8_lossy(&x).to_string(), + ) + }) + .collect(); + } + _ => {} + } + } + } + + vec![] + } +} diff --git a/crates/nu-cli/src/errors.rs b/crates/nu-cli/src/errors.rs index a05a6cb238..bfb29d506b 100644 --- a/crates/nu-cli/src/errors.rs +++ b/crates/nu-cli/src/errors.rs @@ -1,372 +1,53 @@ -use core::ops::Range; +use miette::{LabeledSpan, MietteHandler, ReportHandler, Severity, SourceCode}; +use nu_protocol::engine::StateWorkingSet; +use thiserror::Error; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; -use nu_parser::ParseError; -use nu_protocol::{engine::StateWorkingSet, ShellError, Span}; +/// This error exists so that we can defer SourceCode handling. It simply +/// forwards most methods, except for `.source_code()`, which we provide. +#[derive(Error)] +#[error("{0}")] +struct CliError<'src>( + &'src (dyn miette::Diagnostic + Send + Sync + 'static), + &'src StateWorkingSet<'src>, +); -fn convert_span_to_diag( - working_set: &StateWorkingSet, - span: &Span, -) -> Result<(usize, Range), Box> { - for (file_id, (_, start, end)) in working_set.files().enumerate() { - if span.start >= *start && span.end <= *end { - let new_start = span.start - start; - let new_end = span.end - start; +impl std::fmt::Debug for CliError<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + MietteHandler::default().debug(self, f)?; + Ok(()) + } +} - return Ok((file_id, new_start..new_end)); - } +impl<'src> miette::Diagnostic for CliError<'src> { + fn code<'a>(&'a self) -> Option> { + self.0.code() } - if span.start == working_set.next_span_start() { - // We're trying to highlight the space after the end - if let Some((file_id, (_, _, end))) = working_set.files().enumerate().last() { - return Ok((file_id, *end..(*end + 1))); - } + fn severity(&self) -> Option { + self.0.severity() } - panic!( - "internal error: can't find span in parser state: {:?}", - span - ) + fn help<'a>(&'a self) -> Option> { + self.0.help() + } + + fn url<'a>(&'a self) -> Option> { + self.0.url() + } + + fn labels<'a>(&'a self) -> Option + 'a>> { + self.0.labels() + } + + // Finally, we redirect the source_code method to our own source. + fn source_code(&self) -> Option<&dyn SourceCode> { + Some(&self.1) + } } -pub fn report_parsing_error( +pub fn report_error( working_set: &StateWorkingSet, - error: &ParseError, -) -> Result<(), Box> { - let writer = StandardStream::stderr(ColorChoice::Always); - let config = codespan_reporting::term::Config::default(); - - let diagnostic = - match error { - ParseError::Mismatch(expected, found, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Type mismatch during operation") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("expected {}, found {}", expected, found))]) - } - ParseError::ExtraTokens(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Extra tokens in code") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("extra tokens") - ]) - } - ParseError::ExtraPositional(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Extra positional argument") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("extra positional argument")]) - } - ParseError::UnexpectedEof(s, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unexpected end of code") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("expected {}", s))]) - } - ParseError::Unclosed(delim, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unclosed delimiter") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("unclosed {}", delim))]) - } - ParseError::UnknownStatement(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unknown statement") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("unknown statement") - ]) - } - ParseError::MultipleRestParams(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Multiple rest params") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("multiple rest params")]) - } - ParseError::VariableNotFound(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Variable not found") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("variable not found") - ]) - } - ParseError::UnknownCommand(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unknown command") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("unknown command") - ]) - } - ParseError::UnknownFlag(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unknown flag") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("unknown flag") - ]) - } - ParseError::UnknownType(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unknown type") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("unknown type") - ]) - } - ParseError::MissingFlagParam(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Missing flag param") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("flag missing parameter")]) - } - ParseError::ShortFlagBatchCantTakeArg(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Batches of short flags can't take arguments") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("short flag batches can't take args")]) - } - ParseError::KeywordMissingArgument(name, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message(format!("Missing argument to {}", name)) - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("missing value that follows {}", name))]) - } - ParseError::MissingPositional(name, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Missing required positional arg") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("missing {}", name))]) - } - ParseError::MissingType(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Missing type") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("expected type") - ]) - } - ParseError::TypeMismatch(expected, found, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Type mismatch") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("expected {:?}, found {:?}", expected, found))]) - } - ParseError::MissingRequiredFlag(name, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Missing required flag") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("missing required flag {}", name))]) - } - ParseError::IncompleteMathExpression(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Incomplete math expresssion") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("incomplete math expression")]) - } - ParseError::UnknownState(name, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Unknown state") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("unknown state {}", name))]) - } - ParseError::NonUtf8(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Non-UTF8 code") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("non-UTF8 code") - ]) - } - ParseError::Expected(expected, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Parse mismatch during operation") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("expected {}", expected))]) - } - ParseError::UnsupportedOperation(op_span, lhs_span, lhs_ty, rhs_span, rhs_ty) => { - let (lhs_file_id, lhs_range) = convert_span_to_diag(working_set, lhs_span)?; - let (rhs_file_id, rhs_range) = convert_span_to_diag(working_set, rhs_span)?; - let (op_file_id, op_range) = convert_span_to_diag(working_set, op_span)?; - Diagnostic::error() - .with_message("Unsupported operation") - .with_labels(vec![ - Label::primary(op_file_id, op_range) - .with_message("doesn't support these values"), - Label::secondary(lhs_file_id, lhs_range).with_message(lhs_ty.to_string()), - Label::secondary(rhs_file_id, rhs_range).with_message(rhs_ty.to_string()), - ]) - } - ParseError::ExpectedKeyword(expected, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Expected keyword") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("expected {}", expected))]) - } - ParseError::IncompleteParser(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Parser incomplete") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("parser support missing for this expression")]) - } - ParseError::RestNeedsName(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Rest parameter needs a name") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("needs a parameter name")]) - } - }; - - // println!("DIAG"); - // println!("{:?}", diagnostic); - codespan_reporting::term::emit(&mut writer.lock(), &config, working_set, &diagnostic)?; - - Ok(()) -} - -pub fn report_shell_error( - working_set: &StateWorkingSet, - error: &ShellError, -) -> Result<(), Box> { - let writer = StandardStream::stderr(ColorChoice::Always); - let config = codespan_reporting::term::Config::default(); - - let diagnostic = - match error { - ShellError::OperatorMismatch { - op_span, - lhs_ty, - lhs_span, - rhs_ty, - rhs_span, - } => { - let (lhs_file_id, lhs_range) = convert_span_to_diag(working_set, lhs_span)?; - let (rhs_file_id, rhs_range) = convert_span_to_diag(working_set, rhs_span)?; - let (op_file_id, op_range) = convert_span_to_diag(working_set, op_span)?; - Diagnostic::error() - .with_message("Type mismatch during operation") - .with_labels(vec![ - Label::primary(op_file_id, op_range) - .with_message("type mismatch for operator"), - Label::secondary(lhs_file_id, lhs_range).with_message(lhs_ty.to_string()), - Label::secondary(rhs_file_id, rhs_range).with_message(rhs_ty.to_string()), - ]) - } - ShellError::UnsupportedOperator(op, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message(format!("Unsupported operator: {}", op)) - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("unsupported operator")]) - } - ShellError::UnknownOperator(op, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message(format!("Unsupported operator: {}", op)) - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("unsupported operator")]) - } - ShellError::ExternalNotSupported(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("External commands not yet supported") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("external not supported")]) - } - ShellError::InternalError(s) => { - Diagnostic::error().with_message(format!("Internal error: {}", s)) - } - ShellError::VariableNotFoundAtRuntime(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Variable not found") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("variable not found") - ]) - } - ShellError::CantConvert(s, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message(format!("Can't convert to {}", s)) - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("can't convert to {}", s))]) - } - ShellError::CannotCreateRange(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - Diagnostic::error() - .with_message("Can't convert range to countable values") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("can't convert to countable values")]) - } - ShellError::DivisionByZero(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - - Diagnostic::error() - .with_message("Division by zero") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("division by zero") - ]) - } - ShellError::AccessBeyondEnd(len, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - - Diagnostic::error() - .with_message("Row number too large") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("row number too large (max: {})", *len))]) - } - ShellError::AccessBeyondEndOfStream(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - - Diagnostic::error() - .with_message("Row number too large") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message("row number too large")]) - } - ShellError::IncompatiblePathAccess(name, span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - - Diagnostic::error() - .with_message("Data cannot be accessed with a cell path") - .with_labels(vec![Label::primary(diag_file_id, diag_range) - .with_message(format!("{} doesn't support cell paths", name))]) - } - ShellError::CantFindColumn(span) => { - let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?; - - //FIXME: add "did you mean" - Diagnostic::error() - .with_message("Cannot find column") - .with_labels(vec![ - Label::primary(diag_file_id, diag_range).with_message("cannot find column") - ]) - } - }; - - // println!("DIAG"); - // println!("{:?}", diagnostic); - codespan_reporting::term::emit(&mut writer.lock(), &config, working_set, &diagnostic)?; - - Ok(()) + error: &(dyn miette::Diagnostic + Send + Sync + 'static), +) { + eprintln!("Error: {:?}", CliError(error, working_set)); } diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index a8c602012b..f2643bdfbd 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -1,5 +1,9 @@ +mod completions; mod errors; mod syntax_highlight; +mod validation; -pub use errors::{report_parsing_error, report_shell_error}; +pub use completions::NuCompleter; +pub use errors::report_error; pub use syntax_highlight::NuHighlighter; +pub use validation::NuValidator; diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index 3471b04128..87ec535ed0 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -39,7 +39,9 @@ impl Highlighter for NuHighlighter { [(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)] .to_string(); match shape.1 { + FlatShape::Custom(..) => output.push((Style::new().bold(), next_token)), FlatShape::External => output.push((Style::new().bold(), next_token)), + FlatShape::ExternalArg => output.push((Style::new().bold(), next_token)), FlatShape::Garbage => output.push(( Style::new() .fg(nu_ansi_term::Color::White) diff --git a/crates/nu-cli/src/validation.rs b/crates/nu-cli/src/validation.rs new file mode 100644 index 0000000000..85bf9538ae --- /dev/null +++ b/crates/nu-cli/src/validation.rs @@ -0,0 +1,23 @@ +use std::{cell::RefCell, rc::Rc}; + +use nu_parser::{parse, ParseError}; +use nu_protocol::engine::{EngineState, StateWorkingSet}; +use reedline::{ValidationResult, Validator}; + +pub struct NuValidator { + pub engine_state: Rc>, +} + +impl Validator for NuValidator { + fn validate(&self, line: &str) -> ValidationResult { + let engine_state = self.engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + let (_, err) = parse(&mut working_set, None, line.as_bytes(), false); + + if matches!(err, Some(ParseError::UnexpectedEof(..))) { + ValidationResult::Incomplete + } else { + ValidationResult::Complete + } + } +} diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 1d64aca4d4..beca24e6ee 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -6,6 +6,17 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-protocol = { path = "../nu-protocol" } nu-engine = { path = "../nu-engine" } -nu-parser = {path = "../nu-parser"} \ No newline at end of file +nu-json = { path = "../nu-json" } +nu-protocol = { path = "../nu-protocol" } +<<<<<<< HEAD +nu-engine = { path = "../nu-engine" } +nu-parser = {path = "../nu-parser"} +======= +nu-table = { path = "../nu-table" } + +# Potential dependencies for extras +glob = "0.3.0" +thiserror = "1.0.29" +sysinfo = "0.20.4" +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 diff --git a/crates/nu-command/src/alias.rs b/crates/nu-command/src/core_commands/alias.rs similarity index 100% rename from crates/nu-command/src/alias.rs rename to crates/nu-command/src/core_commands/alias.rs diff --git a/crates/nu-command/src/def.rs b/crates/nu-command/src/core_commands/def.rs similarity index 83% rename from crates/nu-command/src/def.rs rename to crates/nu-command/src/core_commands/def.rs index 25004d800d..0f6ef1b56b 100644 --- a/crates/nu-command/src/def.rs +++ b/crates/nu-command/src/core_commands/def.rs @@ -17,7 +17,11 @@ impl Command for Def { Signature::build("def") .required("def_name", SyntaxShape::String, "definition name") .required("params", SyntaxShape::Signature, "parameters") - .required("block", SyntaxShape::Block, "body of the definition") + .required( + "block", + SyntaxShape::Block(Some(vec![])), + "body of the definition", + ) } fn run( diff --git a/crates/nu-command/src/do_.rs b/crates/nu-command/src/core_commands/do_.rs similarity index 86% rename from crates/nu-command/src/do_.rs rename to crates/nu-command/src/core_commands/do_.rs index 3630e84ff6..20bd0eb7d2 100644 --- a/crates/nu-command/src/do_.rs +++ b/crates/nu-command/src/core_commands/do_.rs @@ -15,7 +15,11 @@ impl Command for Do { } fn signature(&self) -> nu_protocol::Signature { - Signature::build("do").required("block", SyntaxShape::Block, "the block to run") + Signature::build("do").required( + "block", + SyntaxShape::Block(Some(vec![])), + "the block to run", + ) } fn run( diff --git a/crates/nu-command/src/core_commands/help.rs b/crates/nu-command/src/core_commands/help.rs new file mode 100644 index 0000000000..0313f25534 --- /dev/null +++ b/crates/nu-command/src/core_commands/help.rs @@ -0,0 +1,406 @@ +use nu_protocol::{ + ast::Call, + engine::{Command, EvaluationContext}, + Example, ShellError, Signature, Spanned, SyntaxShape, Value, +}; + +use nu_engine::CallExt; + +pub struct Help; + +impl Command for Help { + fn name(&self) -> &str { + "help" + } + + fn signature(&self) -> Signature { + Signature::build("help") + .rest( + "rest", + SyntaxShape::String, + "the name of command to get help on", + ) + .named( + "find", + SyntaxShape::String, + "string to find in command usage", + Some('f'), + ) + } + + fn usage(&self) -> &str { + "Display help information about commands." + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + help(context, call) + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "show all commands and sub-commands", + example: "help commands", + result: None, + }, + Example { + description: "generate documentation", + example: "help generate_docs", + result: None, + }, + Example { + description: "show help for single command", + example: "help match", + result: None, + }, + Example { + description: "show help for single sub-command", + example: "help str lpad", + result: None, + }, + Example { + description: "search for string in command usage", + example: "help --find char", + result: None, + }, + ] + } +} + +fn help(context: &EvaluationContext, call: &Call) -> Result { + let span = call.head; + let find: Option> = call.get_flag(context, "find")?; + let rest: Vec> = call.rest(context, 0)?; + + let full_commands = context.get_commands_info(); + + if let Some(f) = find { + let search_string = f.item; + let mut found_cmds_vec = Vec::new(); + + for cmd in full_commands { + let mut cols = vec![]; + let mut vals = vec![]; + + let key = cmd.name.clone(); + let c = cmd.usage.clone(); + let e = cmd.extra_usage.clone(); + if key.to_lowercase().contains(&search_string) + || c.to_lowercase().contains(&search_string) + || e.to_lowercase().contains(&search_string) + { + cols.push("name".into()); + vals.push(Value::String { val: key, span }); + + cols.push("usage".into()); + vals.push(Value::String { val: c, span }); + + cols.push("extra_usage".into()); + vals.push(Value::String { val: e, span }); + + found_cmds_vec.push(Value::Record { cols, vals, span }); + } + } + + return Ok(Value::List { + vals: found_cmds_vec, + span, + }); + } + + if !rest.is_empty() { + let mut found_cmds_vec = Vec::new(); + + if rest[0].item == "commands" { + for cmd in full_commands { + let mut cols = vec![]; + let mut vals = vec![]; + + let key = cmd.name.clone(); + let c = cmd.usage.clone(); + let e = cmd.extra_usage.clone(); + + cols.push("name".into()); + vals.push(Value::String { val: key, span }); + + cols.push("usage".into()); + vals.push(Value::String { val: c, span }); + + cols.push("extra_usage".into()); + vals.push(Value::String { val: e, span }); + + found_cmds_vec.push(Value::Record { cols, vals, span }); + } + } else { + let mut name = String::new(); + + for r in rest { + if !name.is_empty() { + name.push(' '); + } + name.push_str(&r.item); + } + + for cmd in full_commands { + let mut cols = vec![]; + let mut vals = vec![]; + + let key = cmd.name.clone(); + let c = cmd.usage.clone(); + let e = cmd.extra_usage.clone(); + + if key.starts_with(&name) { + cols.push("name".into()); + vals.push(Value::String { val: key, span }); + + cols.push("usage".into()); + vals.push(Value::String { val: c, span }); + + cols.push("extra_usage".into()); + vals.push(Value::String { val: e, span }); + + found_cmds_vec.push(Value::Record { cols, vals, span }); + } + } + } + Ok(Value::List { + vals: found_cmds_vec, + span, + }) + + // FIXME: the fancy help stuff needs to be reimplemented + /* + if rest[0].item == "commands" { + let mut sorted_names = scope.get_command_names(); + sorted_names.sort(); + + let (mut subcommand_names, command_names) = sorted_names + .into_iter() + // private only commands shouldn't be displayed + .filter(|cmd_name| { + scope + .get_command(cmd_name) + .filter(|command| !command.is_private()) + .is_some() + }) + .partition::, _>(|cmd_name| cmd_name.contains(' ')); + + fn process_name( + dict: &mut TaggedDictBuilder, + cmd_name: &str, + scope: Scope, + rest: Vec>, + name: Tag, + ) -> Result<(), ShellError> { + let document_tag = rest[0].tag.clone(); + let value = command_dict( + scope.get_command(cmd_name).ok_or_else(|| { + ShellError::labeled_error( + format!("Could not load {}", cmd_name), + "could not load command", + document_tag, + ) + })?, + name, + ); + + dict.insert_untagged("name", cmd_name); + dict.insert_untagged( + "description", + value + .get_data_by_key("usage".spanned_unknown()) + .ok_or_else(|| { + ShellError::labeled_error( + "Expected a usage key", + "expected a 'usage' key", + &value.tag, + ) + })? + .as_string()?, + ); + + Ok(()) + } + + fn make_subcommands_table( + subcommand_names: &mut Vec, + cmd_name: &str, + scope: Scope, + rest: Vec>, + name: Tag, + ) -> Result { + let (matching, not_matching) = + subcommand_names.drain(..).partition(|subcommand_name| { + subcommand_name.starts_with(&format!("{} ", cmd_name)) + }); + *subcommand_names = not_matching; + Ok(if !matching.is_empty() { + UntaggedValue::table( + &(matching + .into_iter() + .map(|cmd_name: String| -> Result<_, ShellError> { + let mut short_desc = TaggedDictBuilder::new(name.clone()); + process_name( + &mut short_desc, + &cmd_name, + scope.clone(), + rest.clone(), + name.clone(), + )?; + Ok(short_desc.into_value()) + }) + .collect::, _>>()?[..]), + ) + .into_value(name) + } else { + UntaggedValue::nothing().into_value(name) + }) + } + + let iterator = + command_names + .into_iter() + .map(move |cmd_name| -> Result<_, ShellError> { + let mut short_desc = TaggedDictBuilder::new(name.clone()); + process_name( + &mut short_desc, + &cmd_name, + scope.clone(), + rest.clone(), + name.clone(), + )?; + short_desc.insert_value( + "subcommands", + make_subcommands_table( + &mut subcommand_names, + &cmd_name, + scope.clone(), + rest.clone(), + name.clone(), + )?, + ); + ReturnSuccess::value(short_desc.into_value()) + }); + + Ok(iterator.into_action_stream()) + } else if rest[0].item == "generate_docs" { + Ok(ActionStream::one(ReturnSuccess::value(generate_docs( + &scope, + )))) + } else if rest.len() == 2 { + // Check for a subcommand + let command_name = format!("{} {}", rest[0].item, rest[1].item); + if let Some(command) = scope.get_command(&command_name) { + Ok(ActionStream::one(ReturnSuccess::value( + UntaggedValue::string(get_full_help(command.stream_command(), &scope)) + .into_value(Tag::unknown()), + ))) + } else { + Ok(ActionStream::empty()) + } + } else if let Some(command) = scope.get_command(&rest[0].item) { + Ok(ActionStream::one(ReturnSuccess::value( + UntaggedValue::string(get_full_help(command.stream_command(), &scope)) + .into_value(Tag::unknown()), + ))) + } else { + Err(ShellError::labeled_error( + "Can't find command (use 'help commands' for full list)", + "can't find command", + rest[0].tag.span, + )) + } + */ + } else { + let msg = r#"Welcome to Nushell. + +Here are some tips to help you get started. + * help commands - list all available commands + * help - display help about a particular command + +Nushell works on the idea of a "pipeline". Pipelines are commands connected with the '|' character. +Each stage in the pipeline works together to load, parse, and display information to you. + +[Examples] + +List the files in the current directory, sorted by size: + ls | sort-by size + +Get information about the current system: + sys | get host + +Get the processes on your system actively using CPU: + ps | where cpu > 0 + +You can also learn more at https://www.nushell.sh/book/"#; + + Ok(Value::String { + val: msg.into(), + span, + }) + } +} + +/* +fn for_spec(name: &str, ty: &str, required: bool, tag: impl Into) -> Value { + let tag = tag.into(); + + let mut spec = TaggedDictBuilder::new(tag); + + spec.insert_untagged("name", UntaggedValue::string(name)); + spec.insert_untagged("type", UntaggedValue::string(ty)); + spec.insert_untagged( + "required", + UntaggedValue::string(if required { "yes" } else { "no" }), + ); + + spec.into_value() +} + +pub fn signature_dict(signature: Signature, tag: impl Into) -> Value { + let tag = tag.into(); + let mut sig = TaggedListBuilder::new(&tag); + + for arg in &signature.positional { + let is_required = matches!(arg.0, PositionalType::Mandatory(_, _)); + + sig.push_value(for_spec(arg.0.name(), "argument", is_required, &tag)); + } + + if signature.rest_positional.is_some() { + let is_required = false; + sig.push_value(for_spec("rest", "argument", is_required, &tag)); + } + + for (name, ty) in &signature.named { + match ty.0 { + NamedType::Mandatory(_, _) => sig.push_value(for_spec(name, "flag", true, &tag)), + NamedType::Optional(_, _) => sig.push_value(for_spec(name, "flag", false, &tag)), + NamedType::Switch(_) => sig.push_value(for_spec(name, "switch", false, &tag)), + } + } + + sig.into_value() +} + +fn command_dict(command: Command, tag: impl Into) -> Value { + let tag = tag.into(); + + let mut cmd_dict = TaggedDictBuilder::new(&tag); + + cmd_dict.insert_untagged("name", UntaggedValue::string(command.name())); + + cmd_dict.insert_untagged("type", UntaggedValue::string("Command")); + + cmd_dict.insert_value("signature", signature_dict(command.signature(), tag)); + cmd_dict.insert_untagged("usage", UntaggedValue::string(command.usage())); + + cmd_dict.into_value() +} + +*/ diff --git a/crates/nu-command/src/if_.rs b/crates/nu-command/src/core_commands/if_.rs similarity index 94% rename from crates/nu-command/src/if_.rs rename to crates/nu-command/src/core_commands/if_.rs index bb785c4c2c..a9b1489a66 100644 --- a/crates/nu-command/src/if_.rs +++ b/crates/nu-command/src/core_commands/if_.rs @@ -11,13 +11,13 @@ impl Command for If { } fn usage(&self) -> &str { - "Create a variable and give it a value." + "Conditionally run a block." } fn signature(&self) -> nu_protocol::Signature { Signature::build("if") .required("cond", SyntaxShape::Expression, "condition") - .required("then_block", SyntaxShape::Block, "then block") + .required("then_block", SyntaxShape::Block(Some(vec![])), "then block") .optional( "else", SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), diff --git a/crates/nu-command/src/let_.rs b/crates/nu-command/src/core_commands/let_.rs similarity index 100% rename from crates/nu-command/src/let_.rs rename to crates/nu-command/src/core_commands/let_.rs diff --git a/crates/nu-command/src/core_commands/mod.rs b/crates/nu-command/src/core_commands/mod.rs new file mode 100644 index 0000000000..4d909467d6 --- /dev/null +++ b/crates/nu-command/src/core_commands/mod.rs @@ -0,0 +1,17 @@ +mod alias; +mod def; +mod do_; +mod help; +mod if_; +mod let_; +mod module; +mod use_; + +pub use alias::Alias; +pub use def::Def; +pub use do_::Do; +pub use help::Help; +pub use if_::If; +pub use let_::Let; +pub use module::Module; +pub use use_::Use; diff --git a/crates/nu-command/src/core_commands/module.rs b/crates/nu-command/src/core_commands/module.rs new file mode 100644 index 0000000000..e2cec960d8 --- /dev/null +++ b/crates/nu-command/src/core_commands/module.rs @@ -0,0 +1,34 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Module; + +impl Command for Module { + fn name(&self) -> &str { + "module" + } + + fn usage(&self) -> &str { + "Define a custom module" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("module") + .required("module_name", SyntaxShape::String, "module name") + .required( + "block", + SyntaxShape::Block(Some(vec![])), + "body of the module", + ) + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/core_commands/use_.rs b/crates/nu-command/src/core_commands/use_.rs new file mode 100644 index 0000000000..30b5e3d0b0 --- /dev/null +++ b/crates/nu-command/src/core_commands/use_.rs @@ -0,0 +1,28 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Use; + +impl Command for Use { + fn name(&self) -> &str { + "use" + } + + fn usage(&self) -> &str { + "Use definitions from a module" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("use").required("module_name", SyntaxShape::String, "module name") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index a9278dcc08..d2041573f7 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -2,10 +2,17 @@ use std::{cell::RefCell, rc::Rc}; use nu_protocol::{ engine::{EngineState, StateWorkingSet}, - Signature, SyntaxShape, + Signature, }; +<<<<<<< HEAD use crate::{Alias, Benchmark, BuildString, Def, Do, Each, For, If, Length, Let, LetEnv, Source}; +======= +use crate::{ + Alias, Benchmark, BuildString, Def, Do, Each, External, For, From, FromJson, Git, GitCheckout, + Help, If, Length, Let, LetEnv, Lines, ListGitBranches, Ls, Module, Ps, Sys, Table, Use, Where, +}; +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 pub fn create_default_context() -> Rc> { let engine_state = Rc::new(RefCell::new(EngineState::new())); @@ -13,31 +20,34 @@ pub fn create_default_context() -> Rc> { let engine_state = engine_state.borrow(); let mut working_set = StateWorkingSet::new(&*engine_state); - let sig = - Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); - working_set.add_decl(sig.predeclare()); - - working_set.add_decl(Box::new(If)); - - working_set.add_decl(Box::new(Let)); - - working_set.add_decl(Box::new(LetEnv)); - working_set.add_decl(Box::new(Alias)); - - working_set.add_decl(Box::new(BuildString)); - - working_set.add_decl(Box::new(Def)); - - working_set.add_decl(Box::new(For)); - - working_set.add_decl(Box::new(Each)); - - working_set.add_decl(Box::new(Do)); - working_set.add_decl(Box::new(Benchmark)); - + working_set.add_decl(Box::new(BuildString)); + working_set.add_decl(Box::new(Def)); + working_set.add_decl(Box::new(Do)); + working_set.add_decl(Box::new(Each)); + working_set.add_decl(Box::new(External)); + working_set.add_decl(Box::new(For)); + working_set.add_decl(Box::new(From)); + working_set.add_decl(Box::new(FromJson)); + working_set.add_decl(Box::new(Help)); + working_set.add_decl(Box::new(If)); working_set.add_decl(Box::new(Length)); + working_set.add_decl(Box::new(Let)); + working_set.add_decl(Box::new(LetEnv)); + working_set.add_decl(Box::new(Lines)); + working_set.add_decl(Box::new(Ls)); + working_set.add_decl(Box::new(Module)); + working_set.add_decl(Box::new(Ps)); + working_set.add_decl(Box::new(Sys)); + working_set.add_decl(Box::new(Table)); + working_set.add_decl(Box::new(Use)); + working_set.add_decl(Box::new(Where)); + + // This is a WIP proof of concept + working_set.add_decl(Box::new(ListGitBranches)); + working_set.add_decl(Box::new(Git)); + working_set.add_decl(Box::new(GitCheckout)); working_set.add_decl(Box::new(Source)); @@ -51,6 +61,8 @@ pub fn create_default_context() -> Rc> { working_set.add_decl(sig.predeclare()); let sig = Signature::build("stack"); working_set.add_decl(sig.predeclare()); + let sig = Signature::build("contents"); + working_set.add_decl(sig.predeclare()); working_set.render() }; diff --git a/crates/nu-command/src/let_env.rs b/crates/nu-command/src/env/let_env.rs similarity index 100% rename from crates/nu-command/src/let_env.rs rename to crates/nu-command/src/env/let_env.rs diff --git a/crates/nu-command/src/env/mod.rs b/crates/nu-command/src/env/mod.rs new file mode 100644 index 0000000000..fa42c1b093 --- /dev/null +++ b/crates/nu-command/src/env/mod.rs @@ -0,0 +1,3 @@ +mod let_env; + +pub use let_env::LetEnv; diff --git a/crates/nu-command/src/experimental/git.rs b/crates/nu-command/src/experimental/git.rs new file mode 100644 index 0000000000..5fe8521f39 --- /dev/null +++ b/crates/nu-command/src/experimental/git.rs @@ -0,0 +1,51 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Value}; + +pub struct Git; + +impl Command for Git { + fn name(&self) -> &str { + "git" + } + + fn usage(&self) -> &str { + "Run a block" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("git") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + use std::process::Command as ProcessCommand; + use std::process::Stdio; + + let proc = ProcessCommand::new("git").stdout(Stdio::piped()).spawn(); + + match proc { + Ok(child) => { + match child.wait_with_output() { + Ok(val) => { + let result = val.stdout; + + Ok(Value::string(&String::from_utf8_lossy(&result), call.head)) + } + Err(_err) => { + // FIXME + Ok(Value::nothing()) + } + } + } + Err(_err) => { + // FIXME + Ok(Value::nothing()) + } + } + } +} diff --git a/crates/nu-command/src/experimental/git_checkout.rs b/crates/nu-command/src/experimental/git_checkout.rs new file mode 100644 index 0000000000..143fff96b3 --- /dev/null +++ b/crates/nu-command/src/experimental/git_checkout.rs @@ -0,0 +1,66 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct GitCheckout; + +impl Command for GitCheckout { + fn name(&self) -> &str { + "git checkout" + } + + fn usage(&self) -> &str { + "Checkout a git revision" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("git checkout").required( + "branch", + SyntaxShape::Custom(Box::new(SyntaxShape::String), "list-git-branches".into()), + "the branch to checkout", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + use std::process::Command as ProcessCommand; + use std::process::Stdio; + + let block = &call.positional[0]; + + let out = eval_expression(context, block)?; + + let out = out.as_string()?; + + let proc = ProcessCommand::new("git") + .arg("checkout") + .arg(out) + .stdout(Stdio::piped()) + .spawn(); + + match proc { + Ok(child) => { + match child.wait_with_output() { + Ok(val) => { + let result = val.stdout; + + Ok(Value::string(&String::from_utf8_lossy(&result), call.head)) + } + Err(_err) => { + // FIXME + Ok(Value::nothing()) + } + } + } + Err(_err) => { + // FIXME + Ok(Value::nothing()) + } + } + } +} diff --git a/crates/nu-command/src/experimental/list_git_branches.rs b/crates/nu-command/src/experimental/list_git_branches.rs new file mode 100644 index 0000000000..3a0a148925 --- /dev/null +++ b/crates/nu-command/src/experimental/list_git_branches.rs @@ -0,0 +1,69 @@ +// Note: this is a temporary command that later will be converted into a pipeline + +use std::process::Command as ProcessCommand; +use std::process::Stdio; + +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Value}; + +pub struct ListGitBranches; + +//NOTE: this is not a real implementation :D. It's just a simple one to test with until we port the real one. +impl Command for ListGitBranches { + fn name(&self) -> &str { + "list-git-branches" + } + + fn usage(&self) -> &str { + "List the git branches of the current directory." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("list-git-branches") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let list_branches = ProcessCommand::new("git") + .arg("branch") + .stdout(Stdio::piped()) + .spawn(); + + if let Ok(child) = list_branches { + if let Ok(output) = child.wait_with_output() { + let val = output.stdout; + + let s = String::from_utf8_lossy(&val).to_string(); + + let lines: Vec<_> = s + .lines() + .filter_map(|x| { + if x.starts_with("* ") { + None + } else { + Some(x.trim()) + } + }) + .map(|x| Value::String { + val: x.into(), + span: call.head, + }) + .collect(); + + Ok(Value::List { + vals: lines, + span: call.head, + }) + } else { + Ok(Value::Nothing { span: call.head }) + } + } else { + Ok(Value::Nothing { span: call.head }) + } + } +} diff --git a/crates/nu-command/src/experimental/mod.rs b/crates/nu-command/src/experimental/mod.rs new file mode 100644 index 0000000000..b90a5fd0dd --- /dev/null +++ b/crates/nu-command/src/experimental/mod.rs @@ -0,0 +1,7 @@ +mod git; +mod git_checkout; +mod list_git_branches; + +pub use git::Git; +pub use git_checkout::GitCheckout; +pub use list_git_branches::ListGitBranches; diff --git a/crates/nu-command/src/filesystem/ls.rs b/crates/nu-command/src/filesystem/ls.rs new file mode 100644 index 0000000000..c8b00a8fb5 --- /dev/null +++ b/crates/nu-command/src/filesystem/ls.rs @@ -0,0 +1,93 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; + +pub struct Ls; + +//NOTE: this is not a real implementation :D. It's just a simple one to test with until we port the real one. +impl Command for Ls { + fn name(&self) -> &str { + "ls" + } + + fn usage(&self) -> &str { + "List the files in a directory." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("ls").optional( + "pattern", + SyntaxShape::GlobPattern, + "the glob pattern to use", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let pattern = if let Some(expr) = call.positional.get(0) { + let result = eval_expression(context, expr)?; + result.as_string()? + } else { + "*".into() + }; + + let call_span = call.head; + let glob = glob::glob(&pattern).unwrap(); + + Ok(Value::Stream { + stream: glob + .into_iter() + .map(move |x| match x { + Ok(path) => match std::fs::symlink_metadata(&path) { + Ok(metadata) => { + let is_file = metadata.is_file(); + let is_dir = metadata.is_dir(); + let filesize = metadata.len(); + + Value::Record { + cols: vec!["name".into(), "type".into(), "size".into()], + vals: vec![ + Value::String { + val: path.to_string_lossy().to_string(), + span: call_span, + }, + if is_file { + Value::string("file", call_span) + } else if is_dir { + Value::string("dir", call_span) + } else { + Value::Nothing { span: call_span } + }, + Value::Int { + val: filesize as i64, + span: call_span, + }, + ], + span: call_span, + } + } + Err(_) => Value::Record { + cols: vec!["name".into(), "type".into(), "size".into()], + vals: vec![ + Value::String { + val: path.to_string_lossy().to_string(), + span: call_span, + }, + Value::Nothing { span: call_span }, + Value::Nothing { span: call_span }, + ], + span: call_span, + }, + }, + _ => Value::Nothing { span: call_span }, + }) + .into_value_stream(), + span: call_span, + }) + } +} diff --git a/crates/nu-command/src/filesystem/mod.rs b/crates/nu-command/src/filesystem/mod.rs new file mode 100644 index 0000000000..d7d2f30464 --- /dev/null +++ b/crates/nu-command/src/filesystem/mod.rs @@ -0,0 +1,3 @@ +mod ls; + +pub use ls::Ls; diff --git a/crates/nu-command/src/each.rs b/crates/nu-command/src/filters/each.rs similarity index 63% rename from crates/nu-command/src/each.rs rename to crates/nu-command/src/filters/each.rs index cd2894e9b2..e48cfb719a 100644 --- a/crates/nu-command/src/each.rs +++ b/crates/nu-command/src/filters/each.rs @@ -15,7 +15,13 @@ impl Command for Each { } fn signature(&self) -> nu_protocol::Signature { - Signature::build("each").required("block", SyntaxShape::Block, "the block to run") + Signature::build("each") + .required( + "block", + SyntaxShape::Block(Some(vec![SyntaxShape::Any])), + "the block to run", + ) + .switch("numbered", "iterate with an index", Some('n')) } fn run( @@ -27,20 +33,42 @@ impl Command for Each { let block_id = call.positional[0] .as_block() .expect("internal error: expected block"); + + let numbered = call.has_flag("numbered"); let context = context.clone(); + let span = call.head; match input { Value::Range { val, .. } => Ok(Value::Stream { stream: val .into_iter() - .map(move |x| { + .enumerate() + .map(move |(idx, x)| { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(block_id); let state = context.enter_scope(); + if let Some(var) = block.signature.get_positional(0) { if let Some(var_id) = &var.var_id { - state.add_var(*var_id, x); + if numbered { + state.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + state.add_var(*var_id, x); + } } } @@ -55,14 +83,32 @@ impl Command for Each { Value::List { vals: val, .. } => Ok(Value::Stream { stream: val .into_iter() - .map(move |x| { + .enumerate() + .map(move |(idx, x)| { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(block_id); let state = context.enter_scope(); if let Some(var) = block.signature.get_positional(0) { if let Some(var_id) = &var.var_id { - state.add_var(*var_id, x); + if numbered { + state.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + state.add_var(*var_id, x); + } } } @@ -76,14 +122,32 @@ impl Command for Each { }), Value::Stream { stream, .. } => Ok(Value::Stream { stream: stream - .map(move |x| { + .enumerate() + .map(move |(idx, x)| { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(block_id); let state = context.enter_scope(); if let Some(var) = block.signature.get_positional(0) { if let Some(var_id) = &var.var_id { - state.add_var(*var_id, x); + if numbered { + state.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + state.add_var(*var_id, x); + } } } diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/filters/for_.rs similarity index 63% rename from crates/nu-command/src/for_.rs rename to crates/nu-command/src/filters/for_.rs index 35d806dd56..a3dfa56522 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/filters/for_.rs @@ -1,7 +1,7 @@ use nu_engine::{eval_block, eval_expression}; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EvaluationContext}; -use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; +use nu_protocol::{Example, IntoValueStream, Signature, Span, SyntaxShape, Value}; pub struct For; @@ -29,7 +29,11 @@ impl Command for For { ), "range of the loop", ) - .required("block", SyntaxShape::Block, "the block to run") + .required( + "block", + SyntaxShape::Block(Some(vec![])), + "the block to run", + ) } fn run( @@ -87,4 +91,42 @@ impl Command for For { _ => Ok(Value::nothing()), } } + + fn examples(&self) -> Vec { + let span = Span::unknown(); + vec![ + Example { + description: "Echo the square of each integer", + example: "for x in [1 2 3] { $x * $x }", + result: Some(vec![ + Value::Int { val: 1, span }, + Value::Int { val: 4, span }, + Value::Int { val: 9, span }, + ]), + }, + Example { + description: "Work with elements of a range", + example: "for $x in 1..3 { $x }", + result: Some(vec![ + Value::Int { val: 1, span }, + Value::Int { val: 2, span }, + Value::Int { val: 3, span }, + ]), + }, + Example { + description: "Number each item and echo a message", + example: "for $it in ['bob' 'fred'] --numbered { $\"($it.index) is ($it.item)\" }", + result: Some(vec![ + Value::String { + val: "0 is bob".into(), + span, + }, + Value::String { + val: "0 is fred".into(), + span, + }, + ]), + }, + ] + } } diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/filters/length.rs similarity index 100% rename from crates/nu-command/src/length.rs rename to crates/nu-command/src/filters/length.rs diff --git a/crates/nu-command/src/filters/lines.rs b/crates/nu-command/src/filters/lines.rs new file mode 100644 index 0000000000..74333ff3ef --- /dev/null +++ b/crates/nu-command/src/filters/lines.rs @@ -0,0 +1,92 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{ShellError, Signature, Span, Value, ValueStream}; + +pub struct Lines; + +const SPLIT_CHAR: char = '\n'; + +impl Command for Lines { + fn name(&self) -> &str { + "lines" + } + + fn usage(&self) -> &str { + "Converts input to lines" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("lines") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + match input { + #[allow(clippy::needless_collect)] + // Collect is needed because the string may not live long enough for + // the Rc structure to continue using it. If split could take ownership + // of the split values, then this wouldn't be needed + Value::String { val, span } => { + let lines = val + .split(SPLIT_CHAR) + .map(|s| s.to_string()) + .collect::>(); + + let iter = lines.into_iter().filter_map(move |s| { + if !s.is_empty() { + Some(Value::String { val: s, span }) + } else { + None + } + }); + + Ok(Value::Stream { + stream: ValueStream(Rc::new(RefCell::new(iter))), + span: Span::unknown(), + }) + } + Value::Stream { stream, span: _ } => { + let iter = stream + .into_iter() + .filter_map(|value| { + if let Value::String { val, span } = value { + let inner = val + .split(SPLIT_CHAR) + .filter_map(|s| { + if !s.is_empty() { + Some(Value::String { + val: s.trim().into(), + span, + }) + } else { + None + } + }) + .collect::>(); + + Some(inner) + } else { + None + } + }) + .flatten(); + + Ok(Value::Stream { + stream: ValueStream(Rc::new(RefCell::new(iter))), + span: Span::unknown(), + }) + } + val => Err(ShellError::UnsupportedInput( + format!("Not supported input: {}", val.as_string()?), + call.head, + )), + } + } +} diff --git a/crates/nu-command/src/filters/mod.rs b/crates/nu-command/src/filters/mod.rs new file mode 100644 index 0000000000..143a57b65d --- /dev/null +++ b/crates/nu-command/src/filters/mod.rs @@ -0,0 +1,11 @@ +mod each; +mod for_; +mod length; +mod lines; +mod where_; + +pub use each::Each; +pub use for_::For; +pub use length::Length; +pub use lines::Lines; +pub use where_::Where; diff --git a/crates/nu-command/src/filters/where_.rs b/crates/nu-command/src/filters/where_.rs new file mode 100644 index 0000000000..b876277bb0 --- /dev/null +++ b/crates/nu-command/src/filters/where_.rs @@ -0,0 +1,92 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::{Call, Expr, Expression}; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, ShellError, Signature, SyntaxShape, Value}; + +pub struct Where; + +impl Command for Where { + fn name(&self) -> &str { + "where" + } + + fn usage(&self) -> &str { + "Filter values based on a condition." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let cond = call.positional[0].clone(); + + let context = context.enter_scope(); + + let (var_id, cond) = match cond { + Expression { + expr: Expr::RowCondition(var_id, expr), + .. + } => (var_id, expr), + _ => return Err(ShellError::InternalError("Expected row condition".into())), + }; + + match input { + Value::Stream { stream, span } => { + let output_stream = stream + .filter(move |value| { + context.add_var(var_id, value.clone()); + + let result = eval_expression(&context, &cond); + + match result { + Ok(result) => result.is_true(), + _ => false, + } + }) + .into_value_stream(); + + Ok(Value::Stream { + stream: output_stream, + span, + }) + } + Value::List { vals, span } => { + let output_stream = vals + .into_iter() + .filter(move |value| { + context.add_var(var_id, value.clone()); + + let result = eval_expression(&context, &cond); + + match result { + Ok(result) => result.is_true(), + _ => false, + } + }) + .into_value_stream(); + + Ok(Value::Stream { + stream: output_stream, + span, + }) + } + x => { + context.add_var(var_id, x.clone()); + + let result = eval_expression(&context, &cond)?; + + if result.is_true() { + Ok(x) + } else { + Ok(Value::Nothing { span: call.head }) + } + } + } + } +} diff --git a/crates/nu-command/src/formats/from/command.rs b/crates/nu-command/src/formats/from/command.rs new file mode 100644 index 0000000000..cc2b6b2da8 --- /dev/null +++ b/crates/nu-command/src/formats/from/command.rs @@ -0,0 +1,28 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{ShellError, Signature, Value}; + +pub struct From; + +impl Command for From { + fn name(&self) -> &str { + "from" + } + + fn usage(&self) -> &str { + "Parse a string or binary data into structured data" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("from") + } + + fn run( + &self, + _context: &EvaluationContext, + _call: &Call, + _input: Value, + ) -> Result { + Ok(Value::nothing()) + } +} diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs new file mode 100644 index 0000000000..fa206131f7 --- /dev/null +++ b/crates/nu-command/src/formats/from/json.rs @@ -0,0 +1,111 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, ShellError, Signature, Span, Value}; + +pub struct FromJson; + +impl Command for FromJson { + fn name(&self) -> &str { + "from json" + } + + fn usage(&self) -> &str { + "Convert from json to structured data" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("from json").switch( + "objects", + "treat each line as a separate value", + Some('o'), + ) + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let span = input.span(); + let mut string_input = input.collect_string(); + string_input.push('\n'); + + // TODO: turn this into a structured underline of the nu_json error + if call.has_flag("objects") { + #[allow(clippy::needless_collect)] + let lines: Vec = string_input.lines().map(|x| x.to_string()).collect(); + Ok(Value::Stream { + stream: lines + .into_iter() + .map(move |mut x| { + x.push('\n'); + match convert_string_to_value(x, span) { + Ok(v) => v, + Err(error) => Value::Error { error }, + } + }) + .into_value_stream(), + span, + }) + } else { + convert_string_to_value(string_input, span) + } + } +} + +fn convert_nujson_to_value(value: &nu_json::Value, span: Span) -> Value { + match value { + nu_json::Value::Array(array) => { + let v: Vec = array + .iter() + .map(|x| convert_nujson_to_value(x, span)) + .collect(); + + Value::List { vals: v, span } + } + nu_json::Value::Bool(b) => Value::Bool { val: *b, span }, + nu_json::Value::F64(f) => Value::Float { val: *f, span }, + nu_json::Value::I64(i) => Value::Int { val: *i, span }, + nu_json::Value::Null => Value::Nothing { span }, + nu_json::Value::Object(k) => { + let mut cols = vec![]; + let mut vals = vec![]; + + for item in k { + cols.push(item.0.clone()); + vals.push(convert_nujson_to_value(item.1, span)); + } + + Value::Record { cols, vals, span } + } + nu_json::Value::U64(u) => { + if *u > i64::MAX as u64 { + Value::Error { + error: ShellError::CantConvert("i64 sized integer".into(), span), + } + } else { + Value::Int { + val: *u as i64, + span, + } + } + } + nu_json::Value::String(s) => Value::String { + val: s.clone(), + span, + }, + } +} + +fn convert_string_to_value(string_input: String, span: Span) -> Result { + let result: Result = nu_json::from_str(&string_input); + match result { + Ok(value) => Ok(convert_nujson_to_value(&value, span)), + + Err(_x) => Err(ShellError::CantConvert( + "structured data from json".into(), + span, + )), + } +} diff --git a/crates/nu-command/src/formats/from/mod.rs b/crates/nu-command/src/formats/from/mod.rs new file mode 100644 index 0000000000..78251e562d --- /dev/null +++ b/crates/nu-command/src/formats/from/mod.rs @@ -0,0 +1,5 @@ +mod command; +mod json; + +pub use command::From; +pub use json::FromJson; diff --git a/crates/nu-command/src/formats/mod.rs b/crates/nu-command/src/formats/mod.rs new file mode 100644 index 0000000000..ad1e005635 --- /dev/null +++ b/crates/nu-command/src/formats/mod.rs @@ -0,0 +1,3 @@ +mod from; + +pub use from::*; diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index dad3122926..35b71cfd30 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -1,8 +1,6 @@ -mod alias; -mod benchmark; -mod build_string; -mod def; +mod core_commands; mod default_context; +<<<<<<< HEAD mod do_; mod each; mod for_; @@ -25,3 +23,24 @@ pub use length::Length; pub use let_::Let; pub use let_env::LetEnv; pub use source::Source; +======= +mod env; +mod experimental; +mod filesystem; +mod filters; +mod formats; +mod strings; +mod system; +mod viewers; + +pub use core_commands::*; +pub use default_context::*; +pub use env::*; +pub use experimental::*; +pub use filesystem::*; +pub use filters::*; +pub use formats::*; +pub use strings::*; +pub use system::*; +pub use viewers::*; +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 diff --git a/crates/nu-command/src/build_string.rs b/crates/nu-command/src/strings/build_string.rs similarity index 72% rename from crates/nu-command/src/build_string.rs rename to crates/nu-command/src/strings/build_string.rs index 90516cde07..a0b64b9f3d 100644 --- a/crates/nu-command/src/build_string.rs +++ b/crates/nu-command/src/strings/build_string.rs @@ -1,7 +1,7 @@ use nu_engine::eval_expression; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EvaluationContext}; -use nu_protocol::{Signature, SyntaxShape, Value}; +use nu_protocol::{ShellError, Signature, SyntaxShape, Value}; pub struct BuildString; @@ -24,13 +24,12 @@ impl Command for BuildString { call: &Call, _input: Value, ) -> Result { - let mut output = vec![]; + let output = call + .positional + .iter() + .map(|expr| eval_expression(context, expr).map(|val| val.into_string())) + .collect::, ShellError>>()?; - for expr in &call.positional { - let val = eval_expression(context, expr)?; - - output.push(val.into_string()); - } Ok(Value::String { val: output.join(""), span: call.head, diff --git a/crates/nu-command/src/strings/mod.rs b/crates/nu-command/src/strings/mod.rs new file mode 100644 index 0000000000..8691acce1e --- /dev/null +++ b/crates/nu-command/src/strings/mod.rs @@ -0,0 +1,3 @@ +mod build_string; + +pub use build_string::BuildString; diff --git a/crates/nu-command/src/benchmark.rs b/crates/nu-command/src/system/benchmark.rs similarity index 88% rename from crates/nu-command/src/benchmark.rs rename to crates/nu-command/src/system/benchmark.rs index 5bd5a6afcc..1590e8af84 100644 --- a/crates/nu-command/src/benchmark.rs +++ b/crates/nu-command/src/system/benchmark.rs @@ -17,7 +17,11 @@ impl Command for Benchmark { } fn signature(&self) -> nu_protocol::Signature { - Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run") + Signature::build("benchmark").required( + "block", + SyntaxShape::Block(Some(vec![])), + "the block to run", + ) } fn run( diff --git a/crates/nu-command/src/system/mod.rs b/crates/nu-command/src/system/mod.rs new file mode 100644 index 0000000000..a921225973 --- /dev/null +++ b/crates/nu-command/src/system/mod.rs @@ -0,0 +1,9 @@ +mod benchmark; +mod ps; +mod run_external; +mod sys; + +pub use benchmark::Benchmark; +pub use ps::Ps; +pub use run_external::{External, ExternalCommand}; +pub use sys::Sys; diff --git a/crates/nu-command/src/system/ps.rs b/crates/nu-command/src/system/ps.rs new file mode 100644 index 0000000000..b51310a387 --- /dev/null +++ b/crates/nu-command/src/system/ps.rs @@ -0,0 +1,128 @@ +use nu_protocol::{ + ast::Call, + engine::{Command, EvaluationContext}, + Example, ShellError, Signature, Value, +}; +use sysinfo::{ProcessExt, System, SystemExt}; + +pub struct Ps; + +impl Command for Ps { + fn name(&self) -> &str { + "ps" + } + + fn signature(&self) -> Signature { + Signature::build("ps") + .desc("View information about system processes.") + .switch( + "long", + "list all available columns for each entry", + Some('l'), + ) + .filter() + } + + fn usage(&self) -> &str { + "View information about system processes." + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + run_ps(call) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "List the system processes", + example: "ps", + result: None, + }] + } +} + +fn run_ps(call: &Call) -> Result { + let span = call.head; + let long = call.has_flag("long"); + let mut sys = System::new_all(); + sys.refresh_all(); + + let mut output = vec![]; + + let result: Vec<_> = sys.processes().iter().map(|x| *x.0).collect(); + + for pid in result { + if let Some(result) = sys.process(pid) { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("pid".into()); + vals.push(Value::Int { + val: pid as i64, + span, + }); + + cols.push("name".into()); + vals.push(Value::String { + val: result.name().into(), + span, + }); + + cols.push("status".into()); + vals.push(Value::String { + val: format!("{:?}", result.status()), + span, + }); + + cols.push("cpu".into()); + vals.push(Value::Float { + val: result.cpu_usage() as f64, + span, + }); + + cols.push("mem".into()); + vals.push(Value::Filesize { + val: result.memory() * 1000, + span, + }); + + cols.push("virtual".into()); + vals.push(Value::Filesize { + val: result.virtual_memory() * 1000, + span, + }); + + if long { + cols.push("parent".into()); + if let Some(parent) = result.parent() { + vals.push(Value::Int { + val: parent as i64, + span, + }); + } else { + vals.push(Value::Nothing { span }); + } + + cols.push("exe".into()); + vals.push(Value::String { + val: result.exe().to_string_lossy().to_string(), + span, + }); + + cols.push("command".into()); + vals.push(Value::String { + val: result.cmd().join(" "), + span, + }); + } + + output.push(Value::Record { cols, vals, span }); + } + } + + Ok(Value::List { vals: output, span }) +} diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs new file mode 100644 index 0000000000..8d03b4a7d3 --- /dev/null +++ b/crates/nu-command/src/system/run_external.rs @@ -0,0 +1,281 @@ +use std::borrow::Cow; +use std::cell::RefCell; +use std::env; +use std::io::{BufRead, BufReader, Write}; +use std::process::{ChildStdin, Command as CommandSys, Stdio}; +use std::rc::Rc; +use std::sync::mpsc; + +use nu_protocol::{ + ast::{Call, Expression}, + engine::{Command, EvaluationContext}, + ShellError, Signature, SyntaxShape, Value, +}; +use nu_protocol::{Span, ValueStream}; + +use nu_engine::eval_expression; + +const OUTPUT_BUFFER_SIZE: usize = 8192; + +pub struct External; + +impl Command for External { + fn name(&self) -> &str { + "run_external" + } + + fn usage(&self) -> &str { + "Runs external command" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("run_external") + .switch("last_expression", "last_expression", None) + .rest("rest", SyntaxShape::Any, "external command to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let command = ExternalCommand::try_new(call, context)?; + command.run_with_input(input) + } +} + +pub struct ExternalCommand<'call, 'contex> { + pub name: &'call Expression, + pub args: &'call [Expression], + pub context: &'contex EvaluationContext, + pub last_expression: bool, +} + +impl<'call, 'contex> ExternalCommand<'call, 'contex> { + pub fn try_new( + call: &'call Call, + context: &'contex EvaluationContext, + ) -> Result { + if call.positional.is_empty() { + return Err(ShellError::ExternalNotSupported(call.head)); + } + + Ok(Self { + name: &call.positional[0], + args: &call.positional[1..], + context, + last_expression: call.has_flag("last_expression"), + }) + } + + pub fn get_name(&self) -> Result { + let value = eval_expression(self.context, self.name)?; + value.as_string() + } + + pub fn get_args(&self) -> Vec { + self.args + .iter() + .filter_map(|expr| eval_expression(self.context, expr).ok()) + .filter_map(|value| value.as_string().ok()) + .collect() + } + + pub fn run_with_input(&self, input: Value) -> Result { + let mut process = self.create_command(); + + // TODO. We don't have a way to know the current directory + // This should be information from the EvaluationContex or EngineState + let path = env::current_dir().unwrap(); + process.current_dir(path); + + let envs = self.context.stack.get_env_vars(); + process.envs(envs); + + // If the external is not the last command, its output will get piped + // either as a string or binary + if !self.last_expression { + process.stdout(Stdio::piped()); + } + + // If there is an input from the pipeline. The stdin from the process + // is piped so it can be used to send the input information + if let Value::String { .. } = input { + process.stdin(Stdio::piped()); + } + + if let Value::Stream { .. } = input { + process.stdin(Stdio::piped()); + } + + match process.spawn() { + Err(err) => Err(ShellError::ExternalCommand( + format!("{}", err), + self.name.span, + )), + Ok(mut child) => { + // if there is a string or a stream, that is sent to the pipe std + match input { + Value::String { val, span: _ } => { + if let Some(mut stdin_write) = child.stdin.take() { + self.write_to_stdin(&mut stdin_write, val.as_bytes())? + } + } + Value::Binary { val, span: _ } => { + if let Some(mut stdin_write) = child.stdin.take() { + self.write_to_stdin(&mut stdin_write, &val)? + } + } + Value::Stream { stream, span: _ } => { + if let Some(mut stdin_write) = child.stdin.take() { + for value in stream { + match value { + Value::String { val, span: _ } => { + self.write_to_stdin(&mut stdin_write, val.as_bytes())? + } + Value::Binary { val, span: _ } => { + self.write_to_stdin(&mut stdin_write, &val)? + } + _ => continue, + } + } + } + } + _ => (), + } + + // If this external is not the last expression, then its output is piped to a channel + // and we create a ValueStream that can be consumed + let value = if !self.last_expression { + let (tx, rx) = mpsc::channel(); + let stdout = child.stdout.take().ok_or_else(|| { + ShellError::ExternalCommand( + "Error taking stdout from external".to_string(), + self.name.span, + ) + })?; + + std::thread::spawn(move || { + // Stdout is read using the Buffer reader. It will do so until there is an + // error or there are no more bytes to read + let mut buf_read = BufReader::with_capacity(OUTPUT_BUFFER_SIZE, stdout); + while let Ok(bytes) = buf_read.fill_buf() { + if bytes.is_empty() { + break; + } + + // The Cow generated from the function represents the conversion + // from bytes to String. If no replacements are required, then the + // borrowed value is a proper UTF-8 string. The Owned option represents + // a string where the values had to be replaced, thus marking it as bytes + let data = match String::from_utf8_lossy(bytes) { + Cow::Borrowed(s) => Data::String(s.into()), + Cow::Owned(_) => Data::Bytes(bytes.to_vec()), + }; + + let length = bytes.len(); + buf_read.consume(length); + + match tx.send(data) { + Ok(_) => continue, + Err(_) => break, + } + } + }); + + // The ValueStream is consumed by the next expression in the pipeline + Value::Stream { + stream: ValueStream(Rc::new(RefCell::new(ChannelReceiver::new(rx)))), + span: Span::unknown(), + } + } else { + Value::nothing() + }; + + match child.wait() { + Err(err) => Err(ShellError::ExternalCommand( + format!("{}", err), + self.name.span, + )), + Ok(_) => Ok(value), + } + } + } + } + + fn create_command(&self) -> CommandSys { + // in all the other cases shell out + if cfg!(windows) { + //TODO. This should be modifiable from the config file. + // We could give the option to call from powershell + // for minimal builds cwd is unused + let mut process = CommandSys::new("cmd"); + process.arg("/c"); + process.arg(&self.get_name().unwrap()); + for arg in self.get_args() { + // Clean the args before we use them: + // https://stackoverflow.com/questions/1200235/how-to-pass-a-quoted-pipe-character-to-cmd-exe + // cmd.exe needs to have a caret to escape a pipe + let arg = arg.replace("|", "^|"); + process.arg(&arg); + } + process + } else { + let cmd_with_args = vec![self.get_name().unwrap(), self.get_args().join(" ")].join(" "); + let mut process = CommandSys::new("sh"); + process.arg("-c").arg(cmd_with_args); + process + } + } + + fn write_to_stdin(&self, stdin_write: &mut ChildStdin, val: &[u8]) -> Result<(), ShellError> { + if stdin_write.write(val).is_err() { + Err(ShellError::ExternalCommand( + "Error writing input to stdin".to_string(), + self.name.span, + )) + } else { + Ok(()) + } + } +} + +// The piped data from stdout from the external command can be either String +// or binary. We use this enum to pass the data from the spawned process +enum Data { + String(String), + Bytes(Vec), +} + +// Receiver used for the ValueStream +// It implements iterator so it can be used as a ValueStream +struct ChannelReceiver { + rx: mpsc::Receiver, +} + +impl ChannelReceiver { + pub fn new(rx: mpsc::Receiver) -> Self { + Self { rx } + } +} + +impl Iterator for ChannelReceiver { + type Item = Value; + + fn next(&mut self) -> Option { + match self.rx.recv() { + Ok(v) => match v { + Data::String(s) => Some(Value::String { + val: s, + span: Span::unknown(), + }), + Data::Bytes(b) => Some(Value::Binary { + val: b, + span: Span::unknown(), + }), + }, + Err(_) => None, + } + } +} diff --git a/crates/nu-command/src/system/sys.rs b/crates/nu-command/src/system/sys.rs new file mode 100644 index 0000000000..0ceab55abe --- /dev/null +++ b/crates/nu-command/src/system/sys.rs @@ -0,0 +1,356 @@ +use nu_protocol::{ + ast::Call, + engine::{Command, EvaluationContext}, + Example, ShellError, Signature, Span, Value, +}; +use sysinfo::{ComponentExt, DiskExt, NetworkExt, ProcessorExt, System, SystemExt, UserExt}; + +pub struct Sys; + +impl Command for Sys { + fn name(&self) -> &str { + "sys" + } + + fn signature(&self) -> Signature { + Signature::build("sys") + .desc("View information about the current system.") + .filter() + } + + fn usage(&self) -> &str { + "View information about the system." + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + run_sys(call) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Show info about the system", + example: "sys", + result: None, + }] + } +} + +fn run_sys(call: &Call) -> Result { + let span = call.head; + let mut sys = System::new(); + + let mut headers = vec![]; + let mut values = vec![]; + + if let Some(value) = host(&mut sys, span) { + headers.push("host".into()); + values.push(value); + } + if let Some(value) = cpu(&mut sys, span) { + headers.push("cpu".into()); + values.push(value); + } + if let Some(value) = disks(&mut sys, span) { + headers.push("disks".into()); + values.push(value); + } + if let Some(value) = mem(&mut sys, span) { + headers.push("mem".into()); + values.push(value); + } + if let Some(value) = temp(&mut sys, span) { + headers.push("temp".into()); + values.push(value); + } + if let Some(value) = net(&mut sys, span) { + headers.push("net".into()); + values.push(value); + } + + Ok(Value::Record { + cols: headers, + vals: values, + span, + }) +} + +pub fn trim_cstyle_null(s: String) -> String { + s.trim_matches(char::from(0)).to_string() +} + +pub fn disks(sys: &mut System, span: Span) -> Option { + sys.refresh_disks(); + sys.refresh_disks_list(); + + let mut output = vec![]; + for disk in sys.disks() { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("device".into()); + vals.push(Value::String { + val: trim_cstyle_null(disk.name().to_string_lossy().to_string()), + span, + }); + + cols.push("type".into()); + vals.push(Value::String { + val: trim_cstyle_null(String::from_utf8_lossy(disk.file_system()).to_string()), + span, + }); + + cols.push("mount".into()); + vals.push(Value::String { + val: disk.mount_point().to_string_lossy().to_string(), + span, + }); + + cols.push("total".into()); + vals.push(Value::Filesize { + val: disk.total_space(), + span, + }); + + cols.push("free".into()); + vals.push(Value::Filesize { + val: disk.available_space(), + span, + }); + + output.push(Value::Record { cols, vals, span }); + } + if !output.is_empty() { + Some(Value::List { vals: output, span }) + } else { + None + } +} + +pub fn net(sys: &mut System, span: Span) -> Option { + sys.refresh_networks(); + sys.refresh_networks_list(); + + let mut output = vec![]; + for (iface, data) in sys.networks() { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("name".into()); + vals.push(Value::String { + val: trim_cstyle_null(iface.to_string()), + span, + }); + + cols.push("sent".into()); + vals.push(Value::Filesize { + val: data.total_transmitted(), + span, + }); + + cols.push("recv".into()); + vals.push(Value::Filesize { + val: data.total_received(), + span, + }); + + output.push(Value::Record { cols, vals, span }); + } + if !output.is_empty() { + Some(Value::List { vals: output, span }) + } else { + None + } +} + +pub fn cpu(sys: &mut System, span: Span) -> Option { + sys.refresh_cpu(); + + let mut output = vec![]; + for cpu in sys.processors() { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("name".into()); + vals.push(Value::String { + val: trim_cstyle_null(cpu.name().to_string()), + span, + }); + + cols.push("brand".into()); + vals.push(Value::String { + val: trim_cstyle_null(cpu.brand().to_string()), + span, + }); + + cols.push("freq".into()); + vals.push(Value::Int { + val: cpu.frequency() as i64, + span, + }); + + output.push(Value::Record { cols, vals, span }); + } + if !output.is_empty() { + Some(Value::List { vals: output, span }) + } else { + None + } +} + +pub fn mem(sys: &mut System, span: Span) -> Option { + sys.refresh_memory(); + + let mut cols = vec![]; + let mut vals = vec![]; + + let total_mem = sys.total_memory(); + let free_mem = sys.free_memory(); + let total_swap = sys.total_swap(); + let free_swap = sys.free_swap(); + + cols.push("total".into()); + vals.push(Value::Filesize { + val: total_mem * 1000, + span, + }); + + cols.push("free".into()); + vals.push(Value::Filesize { + val: free_mem * 1000, + span, + }); + + cols.push("swap total".into()); + vals.push(Value::Filesize { + val: total_swap * 1000, + span, + }); + + cols.push("swap free".into()); + vals.push(Value::Filesize { + val: free_swap * 1000, + span, + }); + + Some(Value::Record { cols, vals, span }) +} + +pub fn host(sys: &mut System, span: Span) -> Option { + sys.refresh_users_list(); + + let mut cols = vec![]; + let mut vals = vec![]; + + if let Some(name) = sys.name() { + cols.push("name".into()); + vals.push(Value::String { + val: trim_cstyle_null(name), + span, + }); + } + if let Some(version) = sys.os_version() { + cols.push("os version".into()); + vals.push(Value::String { + val: trim_cstyle_null(version), + span, + }); + } + if let Some(version) = sys.kernel_version() { + cols.push("kernel version".into()); + vals.push(Value::String { + val: trim_cstyle_null(version), + span, + }); + } + if let Some(hostname) = sys.host_name() { + cols.push("hostname".into()); + vals.push(Value::String { + val: trim_cstyle_null(hostname), + span, + }); + } + // dict.insert_untagged( + // "uptime", + // UntaggedValue::duration(1000000000 * sys.uptime() as i64), + // ); + + let mut users = vec![]; + for user in sys.users() { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("name".into()); + vals.push(Value::String { + val: trim_cstyle_null(user.name().to_string()), + span, + }); + + let mut groups = vec![]; + for group in user.groups() { + groups.push(Value::String { + val: trim_cstyle_null(group.to_string()), + span, + }); + } + + cols.push("groups".into()); + vals.push(Value::List { vals: groups, span }); + + users.push(Value::Record { cols, vals, span }); + } + if !users.is_empty() { + cols.push("sessions".into()); + vals.push(Value::List { vals: users, span }); + } + + Some(Value::Record { cols, vals, span }) +} + +pub fn temp(sys: &mut System, span: Span) -> Option { + sys.refresh_components(); + sys.refresh_components_list(); + + let mut output = vec![]; + + for component in sys.components() { + let mut cols = vec![]; + let mut vals = vec![]; + + cols.push("unit".into()); + vals.push(Value::String { + val: component.label().to_string(), + span, + }); + + cols.push("temp".into()); + vals.push(Value::Float { + val: component.temperature() as f64, + span, + }); + + cols.push("high".into()); + vals.push(Value::Float { + val: component.max() as f64, + span, + }); + + if let Some(critical) = component.critical() { + cols.push("critical".into()); + vals.push(Value::Float { + val: critical as f64, + span, + }); + } + output.push(Value::Record { cols, vals, span }); + } + if !output.is_empty() { + Some(Value::List { vals: output, span }) + } else { + None + } +} diff --git a/crates/nu-command/src/viewers/mod.rs b/crates/nu-command/src/viewers/mod.rs new file mode 100644 index 0000000000..0ed6450087 --- /dev/null +++ b/crates/nu-command/src/viewers/mod.rs @@ -0,0 +1,3 @@ +mod table; + +pub use table::Table; diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs new file mode 100644 index 0000000000..2a43bcee26 --- /dev/null +++ b/crates/nu-command/src/viewers/table.rs @@ -0,0 +1,166 @@ +use std::collections::HashMap; + +use nu_protocol::ast::{Call, PathMember}; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Span, Value}; +use nu_table::StyledString; + +pub struct Table; + +//NOTE: this is not a real implementation :D. It's just a simple one to test with until we port the real one. +impl Command for Table { + fn name(&self) -> &str { + "table" + } + + fn usage(&self) -> &str { + "Render the table." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("table") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + match input { + Value::List { vals, .. } => { + let table = convert_to_table(vals); + + if let Some(table) = table { + let result = nu_table::draw_table(&table, 80, &HashMap::new()); + + Ok(Value::String { + val: result, + span: call.head, + }) + } else { + Ok(Value::Nothing { span: call.head }) + } + } + Value::Stream { stream, .. } => { + let table = convert_to_table(stream); + + if let Some(table) = table { + let result = nu_table::draw_table(&table, 80, &HashMap::new()); + + Ok(Value::String { + val: result, + span: call.head, + }) + } else { + Ok(Value::Nothing { span: call.head }) + } + } + Value::Record { cols, vals, .. } => { + let mut output = vec![]; + + for (c, v) in cols.into_iter().zip(vals.into_iter()) { + output.push(vec![ + StyledString { + contents: c, + style: nu_table::TextStyle::default_header(), + }, + StyledString { + contents: v.into_string(), + style: nu_table::TextStyle::default(), + }, + ]) + } + + let table = nu_table::Table { + headers: vec![], + data: output, + theme: nu_table::Theme::rounded(), + }; + + let result = nu_table::draw_table(&table, 80, &HashMap::new()); + + Ok(Value::String { + val: result, + span: call.head, + }) + } + x => Ok(x), + } + } +} + +fn convert_to_table(iter: impl IntoIterator) -> Option { + let mut iter = iter.into_iter().peekable(); + + if let Some(first) = iter.peek() { + let mut headers = first.columns(); + + if !headers.is_empty() { + headers.insert(0, "#".into()); + } + + let mut data = vec![]; + + for (row_num, item) in iter.enumerate() { + let mut row = vec![row_num.to_string()]; + + if headers.is_empty() { + row.push(item.into_string()) + } else { + for header in headers.iter().skip(1) { + let result = match item { + Value::Record { .. } => { + item.clone().follow_cell_path(&[PathMember::String { + val: header.into(), + span: Span::unknown(), + }]) + } + _ => Ok(item.clone()), + }; + + match result { + Ok(value) => row.push(value.into_string()), + Err(_) => row.push(String::new()), + } + } + } + + data.push(row); + } + + Some(nu_table::Table { + headers: headers + .into_iter() + .map(|x| StyledString { + contents: x, + style: nu_table::TextStyle::default_header(), + }) + .collect(), + data: data + .into_iter() + .map(|x| { + x.into_iter() + .enumerate() + .map(|(col, y)| { + if col == 0 { + StyledString { + contents: y, + style: nu_table::TextStyle::default_header(), + } + } else { + StyledString { + contents: y, + style: nu_table::TextStyle::basic_left(), + } + } + }) + .collect::>() + }) + .collect(), + theme: nu_table::Theme::rounded(), + }) + } else { + None + } +} diff --git a/crates/nu-engine/Cargo.toml b/crates/nu-engine/Cargo.toml index 3fbdb06501..6bb588bc0a 100644 --- a/crates/nu-engine/Cargo.toml +++ b/crates/nu-engine/Cargo.toml @@ -5,4 +5,5 @@ edition = "2018" [dependencies] nu-parser = { path = "../nu-parser" } -nu-protocol = { path = "../nu-protocol" } \ No newline at end of file +nu-protocol = { path = "../nu-protocol" } +nu-path = { path = "../nu-path" } \ No newline at end of file diff --git a/crates/nu-engine/src/call_ext.rs b/crates/nu-engine/src/call_ext.rs new file mode 100644 index 0000000000..279a47b38c --- /dev/null +++ b/crates/nu-engine/src/call_ext.rs @@ -0,0 +1,47 @@ +use nu_protocol::{ast::Call, engine::EvaluationContext, ShellError}; + +use crate::{eval_expression, FromValue}; + +pub trait CallExt { + fn get_flag( + &self, + context: &EvaluationContext, + name: &str, + ) -> Result, ShellError>; + + fn rest( + &self, + context: &EvaluationContext, + starting_pos: usize, + ) -> Result, ShellError>; +} + +impl CallExt for Call { + fn get_flag( + &self, + context: &EvaluationContext, + name: &str, + ) -> Result, ShellError> { + if let Some(expr) = self.get_flag_expr(name) { + let result = eval_expression(context, &expr)?; + FromValue::from_value(&result).map(Some) + } else { + Ok(None) + } + } + + fn rest( + &self, + context: &EvaluationContext, + starting_pos: usize, + ) -> Result, ShellError> { + let mut output = vec![]; + + for expr in self.positional.iter().skip(starting_pos) { + let result = eval_expression(context, expr)?; + output.push(FromValue::from_value(&result)?); + } + + Ok(output) + } +} diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 0939cf93a2..0abb4efbc4 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,7 +1,12 @@ use nu_parser::parse; use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; +<<<<<<< HEAD use nu_protocol::engine::{EngineState, EvaluationContext, StateWorkingSet}; use nu_protocol::{Range, ShellError, Span, Value}; +======= +use nu_protocol::engine::EvaluationContext; +use nu_protocol::{Range, ShellError, Span, Type, Value}; +>>>>>>> 3567bbbf32302dbc3cbf97a39b03efa3bd3e8bb5 pub fn eval_operator(op: &Expression) -> Result { match op { @@ -69,6 +74,44 @@ fn eval_call(context: &EvaluationContext, call: &Call, input: Value) -> Result Result { + let engine_state = context.engine_state.borrow(); + + let decl_id = engine_state + .find_decl("run_external".as_bytes()) + .ok_or_else(|| ShellError::ExternalNotSupported(*name))?; + + let command = engine_state.get_decl(decl_id); + + let mut call = Call::new(); + call.positional = [*name] + .iter() + .chain(args.iter()) + .map(|span| { + let contents = engine_state.get_span_contents(span); + let val = String::from_utf8_lossy(contents); + Expression { + expr: Expr::String(val.into()), + span: *span, + ty: Type::String, + custom_completion: None, + } + }) + .collect(); + + if last_expression { + call.named.push(("last_expression".into(), None)) + } + + command.run(context, &call, input) +} + pub fn eval_expression( context: &EvaluationContext, expr: &Expression, @@ -86,13 +129,19 @@ pub fn eval_expression( val: *f, span: expr.span, }), - Expr::Range(from, to, operator) => { - // TODO: Embed the min/max into Range and set max to be the true max + Expr::Range(from, next, to, operator) => { let from = if let Some(f) = from { eval_expression(context, f)? } else { - Value::Int { - val: 0i64, + Value::Nothing { + span: Span::unknown(), + } + }; + + let next = if let Some(s) = next { + eval_expression(context, s)? + } else { + Value::Nothing { span: Span::unknown(), } }; @@ -100,44 +149,29 @@ pub fn eval_expression( let to = if let Some(t) = to { eval_expression(context, t)? } else { - Value::Int { - val: 100i64, + Value::Nothing { span: Span::unknown(), } }; - let range = match (&from, &to) { - (&Value::Int { .. }, &Value::Int { .. }) => Range { - from: from.clone(), - to: to.clone(), - inclusion: operator.inclusion, - }, - (lhs, rhs) => { - return Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: lhs.get_type(), - lhs_span: lhs.span(), - rhs_ty: rhs.get_type(), - rhs_span: rhs.span(), - }) - } - }; - Ok(Value::Range { - val: Box::new(range), + val: Box::new(Range::new(expr.span, from, next, to, operator)?), span: expr.span, }) } Expr::Var(var_id) => context .get_var(*var_id) .map_err(move |_| ShellError::VariableNotFoundAtRuntime(expr.span)), - Expr::FullCellPath(column_path) => { - let value = eval_expression(context, &column_path.head)?; + Expr::FullCellPath(cell_path) => { + let value = eval_expression(context, &cell_path.head)?; - value.follow_cell_path(&column_path.tail) + value.follow_cell_path(&cell_path.tail) } + Expr::RowCondition(_, expr) => eval_expression(context, expr), Expr::Call(call) => eval_call(context, call, Value::nothing()), - Expr::ExternalCall(_, _) => Err(ShellError::ExternalNotSupported(expr.span)), + Expr::ExternalCall(name, args) => { + eval_external(context, name, args, Value::nothing(), true) + } Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }), Expr::BinaryOp(lhs, op, rhs) => { let op_span = op.span; @@ -159,7 +193,6 @@ pub fn eval_expression( x => Err(ShellError::UnsupportedOperator(x, op_span)), } } - Expr::Subexpression(block_id) => { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(*block_id); @@ -219,9 +252,9 @@ pub fn eval_block( block: &Block, mut input: Value, ) -> Result { - for stmt in &block.stmts { + for stmt in block.stmts.iter() { if let Statement::Pipeline(pipeline) = stmt { - for elem in &pipeline.expressions { + for (i, elem) in pipeline.expressions.iter().enumerate() { match elem { Expression { expr: Expr::Call(call), @@ -229,6 +262,18 @@ pub fn eval_block( } => { input = eval_call(context, call, input)?; } + Expression { + expr: Expr::ExternalCall(name, args), + .. + } => { + input = eval_external( + context, + name, + args, + input, + i == pipeline.expressions.len() - 1, + )?; + } elem => { input = eval_expression(context, elem)?; diff --git a/crates/nu-engine/src/from_value.rs b/crates/nu-engine/src/from_value.rs new file mode 100644 index 0000000000..b65f9a5ec5 --- /dev/null +++ b/crates/nu-engine/src/from_value.rs @@ -0,0 +1,265 @@ +// use std::path::PathBuf; + +// use nu_path::expand_path; +use nu_protocol::ShellError; +use nu_protocol::{Range, Spanned, Value}; + +pub trait FromValue: Sized { + fn from_value(v: &Value) -> Result; +} + +impl FromValue for Value { + fn from_value(v: &Value) -> Result { + Ok(v.clone()) + } +} + +impl FromValue for Spanned { + fn from_value(v: &Value) -> Result { + match v { + Value::Int { val, span } => Ok(Spanned { + item: *val, + span: *span, + }), + Value::Filesize { val, span } => Ok(Spanned { + // FIXME: error check that this fits + item: *val as i64, + span: *span, + }), + Value::Duration { val, span } => Ok(Spanned { + // FIXME: error check that this fits + item: *val as i64, + span: *span, + }), + + v => Err(ShellError::CantConvert("integer".into(), v.span())), + } + } +} + +impl FromValue for i64 { + fn from_value(v: &Value) -> Result { + match v { + Value::Int { val, .. } => Ok(*val), + Value::Filesize { val, .. } => Ok( + // FIXME: error check that this fits + *val as i64, + ), + Value::Duration { val, .. } => Ok( + // FIXME: error check that this fits + *val as i64, + ), + + v => Err(ShellError::CantConvert("integer".into(), v.span())), + } + } +} + +impl FromValue for Spanned { + fn from_value(v: &Value) -> Result { + match v { + Value::Int { val, span } => Ok(Spanned { + item: *val as f64, + span: *span, + }), + Value::Float { val, span } => Ok(Spanned { + // FIXME: error check that this fits + item: *val, + span: *span, + }), + + v => Err(ShellError::CantConvert("float".into(), v.span())), + } + } +} + +impl FromValue for f64 { + fn from_value(v: &Value) -> Result { + match v { + Value::Float { val, .. } => Ok(*val), + Value::Int { val, .. } => Ok(*val as f64), + v => Err(ShellError::CantConvert("float".into(), v.span())), + } + } +} + +impl FromValue for String { + fn from_value(v: &Value) -> Result { + // FIXME: we may want to fail a little nicer here + Ok(v.clone().into_string()) + } +} + +impl FromValue for Spanned { + fn from_value(v: &Value) -> Result { + Ok(Spanned { + item: v.clone().into_string(), + span: v.span(), + }) + } +} + +//FIXME +/* +impl FromValue for ColumnPath { + fn from_value(v: &Value) -> Result { + match v { + Value:: => Ok(c.clone()), + v => Err(ShellError::type_error("column path", v.spanned_type_name())), + } + } +} + +impl FromValue for bool { + fn from_value(v: &Value) -> Result { + match v { + Value { + value: UntaggedValue::Primitive(Primitive::Boolean(b)), + .. + } => Ok(*b), + Value { + value: UntaggedValue::Row(_), + .. + } => { + let mut shell_error = ShellError::type_error("boolean", v.spanned_type_name()); + shell_error.notes.push( + "Note: you can access columns using dot. eg) $it.column or (ls).column".into(), + ); + Err(shell_error) + } + v => Err(ShellError::type_error("boolean", v.spanned_type_name())), + } + } +} +*/ + +impl FromValue for Spanned { + fn from_value(v: &Value) -> Result { + match v { + Value::Bool { val, span } => Ok(Spanned { + item: *val, + span: *span, + }), + v => Err(ShellError::CantConvert("bool".into(), v.span())), + } + } +} + +// impl FromValue for DateTime { +// fn from_value(v: &Value) -> Result { +// match v { +// Value { +// value: UntaggedValue::Primitive(Primitive::Date(d)), +// .. +// } => Ok(*d), +// Value { +// value: UntaggedValue::Row(_), +// .. +// } => { +// let mut shell_error = ShellError::type_error("date", v.spanned_type_name()); +// shell_error.notes.push( +// "Note: you can access columns using dot. eg) $it.column or (ls).column".into(), +// ); +// Err(shell_error) +// } +// v => Err(ShellError::type_error("date", v.spanned_type_name())), +// } +// } +// } + +impl FromValue for Range { + fn from_value(v: &Value) -> Result { + match v { + Value::Range { val, .. } => Ok((**val).clone()), + v => Err(ShellError::CantConvert("range".into(), v.span())), + } + } +} + +impl FromValue for Spanned { + fn from_value(v: &Value) -> Result { + match v { + Value::Range { val, span } => Ok(Spanned { + item: (**val).clone(), + span: *span, + }), + v => Err(ShellError::CantConvert("range".into(), v.span())), + } + } +} + +// impl FromValue for Vec { +// fn from_value(v: &Value) -> Result { +// match v { +// Value { +// value: UntaggedValue::Primitive(Primitive::Binary(b)), +// .. +// } => Ok(b.clone()), +// Value { +// value: UntaggedValue::Primitive(Primitive::String(s)), +// .. +// } => Ok(s.bytes().collect()), +// Value { +// value: UntaggedValue::Row(_), +// .. +// } => { +// let mut shell_error = ShellError::type_error("binary data", v.spanned_type_name()); +// shell_error.notes.push( +// "Note: you can access columns using dot. eg) $it.column or (ls).column".into(), +// ); +// Err(shell_error) +// } +// v => Err(ShellError::type_error("binary data", v.spanned_type_name())), +// } +// } +// } + +// impl FromValue for Dictionary { +// fn from_value(v: &Value) -> Result { +// match v { +// Value { +// value: UntaggedValue::Row(r), +// .. +// } => Ok(r.clone()), +// v => Err(ShellError::type_error("row", v.spanned_type_name())), +// } +// } +// } + +// impl FromValue for CapturedBlock { +// fn from_value(v: &Value) -> Result { +// match v { +// Value { +// value: UntaggedValue::Block(b), +// .. +// } => Ok((**b).clone()), +// Value { +// value: UntaggedValue::Row(_), +// .. +// } => { +// let mut shell_error = ShellError::type_error("block", v.spanned_type_name()); +// shell_error.notes.push( +// "Note: you can access columns using dot. eg) $it.column or (ls).column".into(), +// ); +// Err(shell_error) +// } +// v => Err(ShellError::type_error("block", v.spanned_type_name())), +// } +// } +// } + +// impl FromValue for Vec { +// fn from_value(v: &Value) -> Result { +// match v { +// Value { +// value: UntaggedValue::Table(t), +// .. +// } => Ok(t.clone()), +// Value { +// value: UntaggedValue::Row(_), +// .. +// } => Ok(vec![v.clone()]), +// v => Err(ShellError::type_error("table", v.spanned_type_name())), +// } +// } +// } diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index c912ee4772..2cbbbee638 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -1,3 +1,7 @@ +mod call_ext; mod eval; +mod from_value; +pub use call_ext::CallExt; pub use eval::{eval_block, eval_expression, eval_operator}; +pub use from_value::FromValue; diff --git a/crates/nu-json/Cargo.toml b/crates/nu-json/Cargo.toml new file mode 100644 index 0000000000..6f28764526 --- /dev/null +++ b/crates/nu-json/Cargo.toml @@ -0,0 +1,24 @@ +[package] +authors = ["The Nu Project Contributors", "Christian Zangl "] +description = "Fork of serde-hjson" +edition = "2018" +license = "MIT" +name = "nu-json" +version = "0.37.1" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +preserve_order = ["linked-hash-map", "linked-hash-map/serde_impl"] +default = ["preserve_order"] + +[dependencies] +serde = "1.0" +num-traits = "0.2.14" +regex = "^1.0" +lazy_static = "1" +linked-hash-map = { version="0.5", optional=true } + +[dev-dependencies] +nu-path = { version = "0.37.1", path="../nu-path" } +serde_json = "1.0.39" diff --git a/crates/nu-json/LICENSE b/crates/nu-json/LICENSE new file mode 100644 index 0000000000..e6fee54fa8 --- /dev/null +++ b/crates/nu-json/LICENSE @@ -0,0 +1,29 @@ +The MIT License (MIT) + +Copyright (c) 2014 The Rust Project Developers +Copyright (c) 2016 Christian Zangl +Copyright (c) 2020 The Nu Project Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crates/nu-json/src/builder.rs b/crates/nu-json/src/builder.rs new file mode 100644 index 0000000000..a69eee28cd --- /dev/null +++ b/crates/nu-json/src/builder.rs @@ -0,0 +1,115 @@ +use serde::ser; + +use crate::value::{self, Map, Value}; + +/// This structure provides a simple interface for constructing a JSON array. +pub struct ArrayBuilder { + array: Vec, +} + +impl Default for ArrayBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ArrayBuilder { + /// Construct an `ObjectBuilder`. + pub fn new() -> ArrayBuilder { + ArrayBuilder { array: Vec::new() } + } + + /// Return the constructed `Value`. + pub fn unwrap(self) -> Value { + Value::Array(self.array) + } + + /// Insert a value into the array. + pub fn push(mut self, v: T) -> ArrayBuilder { + self.array + .push(value::to_value(&v).expect("failed to serialize")); + self + } + + /// Creates and passes an `ArrayBuilder` into a closure, then inserts the resulting array into + /// this array. + pub fn push_array(mut self, f: F) -> ArrayBuilder + where + F: FnOnce(ArrayBuilder) -> ArrayBuilder, + { + let builder = ArrayBuilder::new(); + self.array.push(f(builder).unwrap()); + self + } + + /// Creates and passes an `ArrayBuilder` into a closure, then inserts the resulting object into + /// this array. + pub fn push_object(mut self, f: F) -> ArrayBuilder + where + F: FnOnce(ObjectBuilder) -> ObjectBuilder, + { + let builder = ObjectBuilder::new(); + self.array.push(f(builder).unwrap()); + self + } +} + +/// This structure provides a simple interface for constructing a JSON object. +pub struct ObjectBuilder { + object: Map, +} + +impl Default for ObjectBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ObjectBuilder { + /// Construct an `ObjectBuilder`. + pub fn new() -> ObjectBuilder { + ObjectBuilder { object: Map::new() } + } + + /// Return the constructed `Value`. + pub fn unwrap(self) -> Value { + Value::Object(self.object) + } + + /// Insert a key-value pair into the object. + pub fn insert(mut self, key: S, value: V) -> ObjectBuilder + where + S: Into, + V: ser::Serialize, + { + self.object.insert( + key.into(), + value::to_value(&value).expect("failed to serialize"), + ); + self + } + + /// Creates and passes an `ObjectBuilder` into a closure, then inserts the resulting array into + /// this object. + pub fn insert_array(mut self, key: S, f: F) -> ObjectBuilder + where + S: Into, + F: FnOnce(ArrayBuilder) -> ArrayBuilder, + { + let builder = ArrayBuilder::new(); + self.object.insert(key.into(), f(builder).unwrap()); + self + } + + /// Creates and passes an `ObjectBuilder` into a closure, then inserts the resulting object into + /// this object. + pub fn insert_object(mut self, key: S, f: F) -> ObjectBuilder + where + S: Into, + F: FnOnce(ObjectBuilder) -> ObjectBuilder, + { + let builder = ObjectBuilder::new(); + self.object.insert(key.into(), f(builder).unwrap()); + self + } +} diff --git a/crates/nu-json/src/de.rs b/crates/nu-json/src/de.rs new file mode 100644 index 0000000000..430aa68b01 --- /dev/null +++ b/crates/nu-json/src/de.rs @@ -0,0 +1,833 @@ +//! Hjson Deserialization +//! +//! This module provides for Hjson deserialization with the type `Deserializer`. + +use std::char; +use std::io; +use std::marker::PhantomData; +use std::str; + +use serde::de; + +use super::error::{Error, ErrorCode, Result}; +use super::util::StringReader; +use super::util::{Number, ParseNumber}; + +enum State { + Normal, + Root, + Keyname, +} + +/// A structure that deserializes Hjson into Rust values. +pub struct Deserializer> { + rdr: StringReader, + str_buf: Vec, + state: State, +} + +// macro_rules! try_or_invalid { +// ($self_:expr, $e:expr) => { +// match $e { +// Some(v) => v, +// None => { return Err($self_.error(ErrorCode::InvalidNumber)); } +// } +// } +// } + +impl Deserializer +where + Iter: Iterator, +{ + /// Creates the Hjson parser from an `std::iter::Iterator`. + #[inline] + pub fn new(rdr: Iter) -> Deserializer { + Deserializer { + rdr: StringReader::new(rdr), + str_buf: Vec::with_capacity(128), + state: State::Normal, + } + } + + /// Creates the Hjson parser from an `std::iter::Iterator`. + #[inline] + pub fn new_for_root(rdr: Iter) -> Deserializer { + let mut res = Deserializer::new(rdr); + res.state = State::Root; + res + } + + /// The `Deserializer::end` method should be called after a value has been fully deserialized. + /// This allows the `Deserializer` to validate that the input stream is at the end or that it + /// only has trailing whitespace. + #[inline] + pub fn end(&mut self) -> Result<()> { + self.rdr.parse_whitespace()?; + if self.rdr.eof()? { + Ok(()) + } else { + Err(self.rdr.error(ErrorCode::TrailingCharacters)) + } + } + + fn is_punctuator_char(&mut self, ch: u8) -> bool { + matches!(ch, b'{' | b'}' | b'[' | b']' | b',' | b':') + } + + fn parse_keyname<'de, V>(&mut self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + // quotes for keys are optional in Hjson + // unless they include {}[],: or whitespace. + // assume whitespace was already eaten + + self.str_buf.clear(); + + let mut space: Option = None; + loop { + let ch = self.rdr.next_char_or_null()?; + + if ch == b':' { + if self.str_buf.is_empty() { + return Err(self.rdr.error(ErrorCode::Custom( + "Found ':' but no key name (for an empty key name use quotes)".to_string(), + ))); + } else if space.is_some() + && space.expect("Internal error: json parsing") != self.str_buf.len() + { + return Err(self.rdr.error(ErrorCode::Custom( + "Found whitespace in your key name (use quotes to include)".to_string(), + ))); + } + self.rdr.uneat_char(ch); + let s = str::from_utf8(&self.str_buf).expect("Internal error: json parsing"); + return visitor.visit_str(s); + } else if ch <= b' ' { + if ch == 0 { + return Err(self.rdr.error(ErrorCode::EofWhileParsingObject)); + } else if space.is_none() { + space = Some(self.str_buf.len()); + } + } else if self.is_punctuator_char(ch) { + return Err(self.rdr.error(ErrorCode::Custom("Found a punctuator where a key name was expected (check your syntax or use quotes if the key name includes {}[],: or whitespace)".to_string()))); + } else { + self.str_buf.push(ch); + } + } + } + + fn parse_value<'de, V>(&mut self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + self.rdr.parse_whitespace()?; + + if self.rdr.eof()? { + return Err(self.rdr.error(ErrorCode::EofWhileParsingValue)); + } + + match self.state { + State::Keyname => { + self.state = State::Normal; + return self.parse_keyname(visitor); + } + State::Root => { + self.state = State::Normal; + return self.visit_map(true, visitor); + } + _ => {} + } + + match self.rdr.peek_or_null()? { + /* + b'-' => { + self.rdr.eat_char(); + self.parse_integer(false, visitor) + } + b'0' ... b'9' => { + self.parse_integer(true, visitor) + } + */ + b'"' => { + self.rdr.eat_char(); + self.parse_string()?; + let s = str::from_utf8(&self.str_buf).expect("Internal error: json parsing"); + visitor.visit_str(s) + } + b'[' => { + self.rdr.eat_char(); + let ret = visitor.visit_seq(SeqVisitor::new(self))?; + self.rdr.parse_whitespace()?; + match self.rdr.next_char()? { + Some(b']') => Ok(ret), + Some(_) => Err(self.rdr.error(ErrorCode::TrailingCharacters)), + None => Err(self.rdr.error(ErrorCode::EofWhileParsingList)), + } + } + b'{' => { + self.rdr.eat_char(); + self.visit_map(false, visitor) + } + b'\x00' => Err(self.rdr.error(ErrorCode::ExpectedSomeValue)), + _ => self.parse_tfnns(visitor), + } + } + + fn visit_map<'de, V>(&mut self, root: bool, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + let ret = visitor.visit_map(MapVisitor::new(self, root))?; + self.rdr.parse_whitespace()?; + match self.rdr.next_char()? { + Some(b'}') => { + if !root { + Ok(ret) + } else { + Err(self.rdr.error(ErrorCode::TrailingCharacters)) + } // todo + } + Some(_) => Err(self.rdr.error(ErrorCode::TrailingCharacters)), + None => { + if root { + Ok(ret) + } else { + Err(self.rdr.error(ErrorCode::EofWhileParsingObject)) + } + } + } + } + + fn parse_ident(&mut self, ident: &[u8]) -> Result<()> { + for c in ident { + if Some(*c) != self.rdr.next_char()? { + return Err(self.rdr.error(ErrorCode::ExpectedSomeIdent)); + } + } + + Ok(()) + } + + fn parse_tfnns<'de, V>(&mut self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + // Hjson strings can be quoteless + // returns string, true, false, or null. + self.str_buf.clear(); + + let first = self.rdr.peek()?.expect("Internal error: json parsing"); + + if self.is_punctuator_char(first) { + return Err(self.rdr.error(ErrorCode::PunctuatorInQlString)); + } + + loop { + let ch = self.rdr.next_char_or_null()?; + + let is_eol = ch == b'\r' || ch == b'\n' || ch == b'\x00'; + let is_comment = ch == b'#' + || if ch == b'/' { + let next = self.rdr.peek_or_null()?; + next == b'/' || next == b'*' + } else { + false + }; + if is_eol || is_comment || ch == b',' || ch == b'}' || ch == b']' { + let chf = self.str_buf[0]; + match chf { + b'f' => { + if str::from_utf8(&self.str_buf) + .expect("Internal error: json parsing") + .trim() + == "false" + { + self.rdr.uneat_char(ch); + return visitor.visit_bool(false); + } + } + b'n' => { + if str::from_utf8(&self.str_buf) + .expect("Internal error: json parsing") + .trim() + == "null" + { + self.rdr.uneat_char(ch); + return visitor.visit_unit(); + } + } + b't' => { + if str::from_utf8(&self.str_buf) + .expect("Internal error: json parsing") + .trim() + == "true" + { + self.rdr.uneat_char(ch); + return visitor.visit_bool(true); + } + } + _ => { + if chf == b'-' || (b'0'..=b'9').contains(&chf) { + let mut pn = ParseNumber::new(self.str_buf.iter().copied()); + match pn.parse(false) { + Ok(Number::F64(v)) => { + self.rdr.uneat_char(ch); + return visitor.visit_f64(v); + } + Ok(Number::U64(v)) => { + self.rdr.uneat_char(ch); + return visitor.visit_u64(v); + } + Ok(Number::I64(v)) => { + self.rdr.uneat_char(ch); + return visitor.visit_i64(v); + } + Err(_) => {} // not a number, continue + } + } + } + } + if is_eol { + // remove any whitespace at the end (ignored in quoteless strings) + return visitor.visit_str( + str::from_utf8(&self.str_buf) + .expect("Internal error: json parsing") + .trim(), + ); + } + } + self.str_buf.push(ch); + + if self.str_buf == b"'''" { + return self.parse_ml_string(visitor); + } + } + } + + fn decode_hex_escape(&mut self) -> Result { + let mut i = 0; + let mut n = 0u16; + while i < 4 && !self.rdr.eof()? { + n = match self.rdr.next_char_or_null()? { + c @ b'0'..=b'9' => n * 16_u16 + ((c as u16) - (b'0' as u16)), + b'a' | b'A' => n * 16_u16 + 10_u16, + b'b' | b'B' => n * 16_u16 + 11_u16, + b'c' | b'C' => n * 16_u16 + 12_u16, + b'd' | b'D' => n * 16_u16 + 13_u16, + b'e' | b'E' => n * 16_u16 + 14_u16, + b'f' | b'F' => n * 16_u16 + 15_u16, + _ => { + return Err(self.rdr.error(ErrorCode::InvalidEscape)); + } + }; + + i += 1; + } + + // Error out if we didn't parse 4 digits. + if i != 4 { + return Err(self.rdr.error(ErrorCode::InvalidEscape)); + } + + Ok(n) + } + + fn ml_skip_white(&mut self) -> Result { + match self.rdr.peek_or_null()? { + b' ' | b'\t' | b'\r' => { + self.rdr.eat_char(); + Ok(true) + } + _ => Ok(false), + } + } + + fn ml_skip_indent(&mut self, indent: usize) -> Result<()> { + let mut skip = indent; + while self.ml_skip_white()? && skip > 0 { + skip -= 1; + } + Ok(()) + } + + fn parse_ml_string<'de, V>(&mut self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + self.str_buf.clear(); + + // Parse a multiline string value. + let mut triple = 0; + + // we are at ''' +1 - get indent + let (_, col) = self.rdr.pos(); + let indent = col - 4; + + // skip white/to (newline) + while self.ml_skip_white()? {} + if self.rdr.peek_or_null()? == b'\n' { + self.rdr.eat_char(); + self.ml_skip_indent(indent)?; + } + + // When parsing multiline string values, we must look for ' characters. + loop { + if self.rdr.eof()? { + return Err(self.rdr.error(ErrorCode::EofWhileParsingString)); + } // todo error("Bad multiline string"); + let ch = self.rdr.next_char_or_null()?; + + if ch == b'\'' { + triple += 1; + if triple == 3 { + if self.str_buf.last() == Some(&b'\n') { + self.str_buf.pop(); + } + let res = str::from_utf8(&self.str_buf).expect("Internal error: json parsing"); + //todo if (self.str_buf.slice(-1) === '\n') self.str_buf=self.str_buf.slice(0, -1); // remove last EOL + return visitor.visit_str(res); + } else { + continue; + } + } + + while triple > 0 { + self.str_buf.push(b'\''); + triple -= 1; + } + + if ch != b'\r' { + self.str_buf.push(ch); + } + if ch == b'\n' { + self.ml_skip_indent(indent)?; + } + } + } + + fn parse_string(&mut self) -> Result<()> { + self.str_buf.clear(); + + loop { + let ch = match self.rdr.next_char()? { + Some(ch) => ch, + None => { + return Err(self.rdr.error(ErrorCode::EofWhileParsingString)); + } + }; + + match ch { + b'"' => { + return Ok(()); + } + b'\\' => { + let ch = match self.rdr.next_char()? { + Some(ch) => ch, + None => { + return Err(self.rdr.error(ErrorCode::EofWhileParsingString)); + } + }; + + match ch { + b'"' => self.str_buf.push(b'"'), + b'\\' => self.str_buf.push(b'\\'), + b'/' => self.str_buf.push(b'/'), + b'b' => self.str_buf.push(b'\x08'), + b'f' => self.str_buf.push(b'\x0c'), + b'n' => self.str_buf.push(b'\n'), + b'r' => self.str_buf.push(b'\r'), + b't' => self.str_buf.push(b'\t'), + b'u' => { + let c = match self.decode_hex_escape()? { + 0xDC00..=0xDFFF => { + return Err(self + .rdr + .error(ErrorCode::LoneLeadingSurrogateInHexEscape)); + } + + // Non-BMP characters are encoded as a sequence of + // two hex escapes, representing UTF-16 surrogates. + n1 @ 0xD800..=0xDBFF => { + match (self.rdr.next_char()?, self.rdr.next_char()?) { + (Some(b'\\'), Some(b'u')) => (), + _ => { + return Err(self + .rdr + .error(ErrorCode::UnexpectedEndOfHexEscape)); + } + } + + let n2 = self.decode_hex_escape()?; + + if !(0xDC00..=0xDFFF).contains(&n2) { + return Err(self + .rdr + .error(ErrorCode::LoneLeadingSurrogateInHexEscape)); + } + + let n = (((n1 - 0xD800) as u32) << 10 | (n2 - 0xDC00) as u32) + + 0x1_0000; + + match char::from_u32(n as u32) { + Some(c) => c, + None => { + return Err(self + .rdr + .error(ErrorCode::InvalidUnicodeCodePoint)); + } + } + } + + n => match char::from_u32(n as u32) { + Some(c) => c, + None => { + return Err(self + .rdr + .error(ErrorCode::InvalidUnicodeCodePoint)); + } + }, + }; + + self.str_buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes()); + } + _ => { + return Err(self.rdr.error(ErrorCode::InvalidEscape)); + } + } + } + ch => { + self.str_buf.push(ch); + } + } + } + } + + fn parse_object_colon(&mut self) -> Result<()> { + self.rdr.parse_whitespace()?; + + match self.rdr.next_char()? { + Some(b':') => Ok(()), + Some(_) => Err(self.rdr.error(ErrorCode::ExpectedColon)), + None => Err(self.rdr.error(ErrorCode::EofWhileParsingObject)), + } + } +} + +impl<'de, 'a, Iter> de::Deserializer<'de> for &'a mut Deserializer +where + Iter: Iterator, +{ + type Error = Error; + + #[inline] + fn deserialize_any(self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + if let State::Root = self.state {} + + self.parse_value(visitor) + } + + /// Parses a `null` as a None, and any other values as a `Some(...)`. + #[inline] + fn deserialize_option(self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + self.rdr.parse_whitespace()?; + + match self.rdr.peek_or_null()? { + b'n' => { + self.rdr.eat_char(); + self.parse_ident(b"ull")?; + visitor.visit_none() + } + _ => visitor.visit_some(self), + } + } + + /// Parses a newtype struct as the underlying value. + #[inline] + fn deserialize_newtype_struct(self, _name: &str, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string + bytes byte_buf unit unit_struct seq tuple map + tuple_struct struct enum identifier ignored_any + } +} + +struct SeqVisitor<'a, Iter: 'a + Iterator> { + de: &'a mut Deserializer, +} + +impl<'a, Iter: Iterator> SeqVisitor<'a, Iter> { + fn new(de: &'a mut Deserializer) -> Self { + SeqVisitor { de } + } +} + +impl<'de, 'a, Iter> de::SeqAccess<'de> for SeqVisitor<'a, Iter> +where + Iter: Iterator, +{ + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: de::DeserializeSeed<'de>, + { + self.de.rdr.parse_whitespace()?; + + match self.de.rdr.peek()? { + Some(b']') => { + return Ok(None); + } + Some(_) => {} + None => { + return Err(self.de.rdr.error(ErrorCode::EofWhileParsingList)); + } + } + + let value = seed.deserialize(&mut *self.de)?; + + // in Hjson the comma is optional and trailing commas are allowed + self.de.rdr.parse_whitespace()?; + if self.de.rdr.peek()? == Some(b',') { + self.de.rdr.eat_char(); + self.de.rdr.parse_whitespace()?; + } + + Ok(Some(value)) + } +} + +struct MapVisitor<'a, Iter: 'a + Iterator> { + de: &'a mut Deserializer, + first: bool, + root: bool, +} + +impl<'a, Iter: Iterator> MapVisitor<'a, Iter> { + fn new(de: &'a mut Deserializer, root: bool) -> Self { + MapVisitor { + de, + first: true, + root, + } + } +} + +impl<'de, 'a, Iter> de::MapAccess<'de> for MapVisitor<'a, Iter> +where + Iter: Iterator, +{ + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: de::DeserializeSeed<'de>, + { + self.de.rdr.parse_whitespace()?; + + if self.first { + self.first = false; + } else if self.de.rdr.peek()? == Some(b',') { + // in Hjson the comma is optional and trailing commas are allowed + self.de.rdr.eat_char(); + self.de.rdr.parse_whitespace()?; + } + + match self.de.rdr.peek()? { + Some(b'}') => return Ok(None), // handled later for root + Some(_) => {} + None => { + if self.root { + return Ok(None); + } else { + return Err(self.de.rdr.error(ErrorCode::EofWhileParsingObject)); + } + } + } + + match self.de.rdr.peek()? { + Some(ch) => { + self.de.state = if ch == b'"' { + State::Normal + } else { + State::Keyname + }; + Ok(Some(seed.deserialize(&mut *self.de)?)) + } + None => Err(self.de.rdr.error(ErrorCode::EofWhileParsingValue)), + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + self.de.parse_object_colon()?; + + seed.deserialize(&mut *self.de) + } +} + +impl<'de, 'a, Iter> de::VariantAccess<'de> for &'a mut Deserializer +where + Iter: Iterator, +{ + type Error = Error; + + fn unit_variant(self) -> Result<()> { + de::Deserialize::deserialize(self) + } + + fn newtype_variant_seed(self, seed: T) -> Result + where + T: de::DeserializeSeed<'de>, + { + seed.deserialize(self) + } + + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + de::Deserializer::deserialize_any(self, visitor) + } + + fn struct_variant(self, _fields: &'static [&'static str], visitor: V) -> Result + where + V: de::Visitor<'de>, + { + de::Deserializer::deserialize_any(self, visitor) + } +} + +////////////////////////////////////////////////////////////////////////////// + +/// Iterator that deserializes a stream into multiple Hjson values. +pub struct StreamDeserializer +where + Iter: Iterator, + T: de::DeserializeOwned, +{ + deser: Deserializer, + _marker: PhantomData, +} + +impl StreamDeserializer +where + Iter: Iterator, + T: de::DeserializeOwned, +{ + /// Returns an `Iterator` of decoded Hjson values from an iterator over + /// `Iterator`. + pub fn new(iter: Iter) -> StreamDeserializer { + StreamDeserializer { + deser: Deserializer::new(iter), + _marker: PhantomData, + } + } +} + +impl Iterator for StreamDeserializer +where + Iter: Iterator, + T: de::DeserializeOwned, +{ + type Item = Result; + + fn next(&mut self) -> Option> { + // skip whitespaces, if any + // this helps with trailing whitespaces, since whitespaces between + // values are handled for us. + if let Err(e) = self.deser.rdr.parse_whitespace() { + return Some(Err(e)); + }; + + match self.deser.rdr.eof() { + Ok(true) => None, + Ok(false) => match de::Deserialize::deserialize(&mut self.deser) { + Ok(v) => Some(Ok(v)), + Err(e) => Some(Err(e)), + }, + Err(e) => Some(Err(e)), + } + } +} + +////////////////////////////////////////////////////////////////////////////// + +/// Decodes a Hjson value from an iterator over an iterator +/// `Iterator`. +pub fn from_iter(iter: I) -> Result +where + I: Iterator>, + T: de::DeserializeOwned, +{ + let fold: io::Result> = iter.collect(); + + if let Err(e) = fold { + return Err(Error::Io(e)); + } + + let bytes = fold.expect("Internal error: json parsing"); + + // deserialize tries first to decode with legacy support (new_for_root) + // and then with the standard method if this fails. + // todo: add compile switch + + // deserialize and make sure the whole stream has been consumed + let mut de = Deserializer::new_for_root(bytes.iter().copied()); + de::Deserialize::deserialize(&mut de) + .and_then(|x| de.end().map(|()| x)) + .or_else(|_| { + let mut de2 = Deserializer::new(bytes.iter().copied()); + de::Deserialize::deserialize(&mut de2).and_then(|x| de2.end().map(|()| x)) + }) + + /* without legacy support: + // deserialize and make sure the whole stream has been consumed + let mut de = Deserializer::new(bytes.iter().map(|b| *b)); + let value = match de::Deserialize::deserialize(&mut de) + .and_then(|x| { try!(de.end()); Ok(x) }) + { + Ok(v) => Ok(v), + Err(e) => Err(e), + }; + */ +} + +/// Decodes a Hjson value from a `std::io::Read`. +pub fn from_reader(rdr: R) -> Result +where + R: io::Read, + T: de::DeserializeOwned, +{ + from_iter(rdr.bytes()) +} + +/// Decodes a Hjson value from a byte slice `&[u8]`. +pub fn from_slice(v: &[u8]) -> Result +where + T: de::DeserializeOwned, +{ + from_iter(v.iter().map(|&byte| Ok(byte))) +} + +/// Decodes a Hjson value from a `&str`. +pub fn from_str(s: &str) -> Result +where + T: de::DeserializeOwned, +{ + from_slice(s.as_bytes()) +} diff --git a/crates/nu-json/src/error.rs b/crates/nu-json/src/error.rs new file mode 100644 index 0000000000..33d417c62a --- /dev/null +++ b/crates/nu-json/src/error.rs @@ -0,0 +1,166 @@ +//! JSON Errors +//! +//! This module is centered around the `Error` and `ErrorCode` types, which represents all possible +//! `serde_hjson` errors. + +use std::error; +use std::fmt; +use std::io; +use std::result; +use std::string::FromUtf8Error; + +use serde::de; +use serde::ser; + +/// The errors that can arise while parsing a JSON stream. +#[derive(Clone, PartialEq)] +pub enum ErrorCode { + /// Catchall for syntax error messages + Custom(String), + + /// EOF while parsing a list. + EofWhileParsingList, + + /// EOF while parsing an object. + EofWhileParsingObject, + + /// EOF while parsing a string. + EofWhileParsingString, + + /// EOF while parsing a JSON value. + EofWhileParsingValue, + + /// Expected this character to be a `':'`. + ExpectedColon, + + /// Expected this character to be either a `','` or a `]`. + ExpectedListCommaOrEnd, + + /// Expected this character to be either a `','` or a `}`. + ExpectedObjectCommaOrEnd, + + /// Expected to parse either a `true`, `false`, or a `null`. + ExpectedSomeIdent, + + /// Expected this character to start a JSON value. + ExpectedSomeValue, + + /// Invalid hex escape code. + InvalidEscape, + + /// Invalid number. + InvalidNumber, + + /// Invalid Unicode code point. + InvalidUnicodeCodePoint, + + /// Object key is not a string. + KeyMustBeAString, + + /// Lone leading surrogate in hex escape. + LoneLeadingSurrogateInHexEscape, + + /// JSON has non-whitespace trailing characters after the value. + TrailingCharacters, + + /// Unexpected end of hex escape. + UnexpectedEndOfHexEscape, + + /// Found a punctuator character when expecting a quoteless string. + PunctuatorInQlString, +} + +impl fmt::Debug for ErrorCode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + //use std::fmt::Debug; + + match *self { + ErrorCode::Custom(ref msg) => write!(f, "{}", msg), + ErrorCode::EofWhileParsingList => "EOF while parsing a list".fmt(f), + ErrorCode::EofWhileParsingObject => "EOF while parsing an object".fmt(f), + ErrorCode::EofWhileParsingString => "EOF while parsing a string".fmt(f), + ErrorCode::EofWhileParsingValue => "EOF while parsing a value".fmt(f), + ErrorCode::ExpectedColon => "expected `:`".fmt(f), + ErrorCode::ExpectedListCommaOrEnd => "expected `,` or `]`".fmt(f), + ErrorCode::ExpectedObjectCommaOrEnd => "expected `,` or `}`".fmt(f), + ErrorCode::ExpectedSomeIdent => "expected ident".fmt(f), + ErrorCode::ExpectedSomeValue => "expected value".fmt(f), + ErrorCode::InvalidEscape => "invalid escape".fmt(f), + ErrorCode::InvalidNumber => "invalid number".fmt(f), + ErrorCode::InvalidUnicodeCodePoint => "invalid Unicode code point".fmt(f), + ErrorCode::KeyMustBeAString => "key must be a string".fmt(f), + ErrorCode::LoneLeadingSurrogateInHexEscape => { + "lone leading surrogate in hex escape".fmt(f) + } + ErrorCode::TrailingCharacters => "trailing characters".fmt(f), + ErrorCode::UnexpectedEndOfHexEscape => "unexpected end of hex escape".fmt(f), + ErrorCode::PunctuatorInQlString => { + "found a punctuator character when expecting a quoteless string".fmt(f) + } + } + } +} + +/// This type represents all possible errors that can occur when serializing or deserializing a +/// value into JSON. +#[derive(Debug)] +pub enum Error { + /// The JSON value had some syntactic error. + Syntax(ErrorCode, usize, usize), + + /// Some IO error occurred when serializing or deserializing a value. + Io(io::Error), + + /// Some UTF8 error occurred while serializing or deserializing a value. + FromUtf8(FromUtf8Error), +} + +impl error::Error for Error { + fn cause(&self) -> Option<&dyn error::Error> { + match *self { + Error::Io(ref error) => Some(error), + Error::FromUtf8(ref error) => Some(error), + _ => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match *self { + Error::Syntax(ref code, line, col) => { + write!(fmt, "{:?} at line {} column {}", code, line, col) + } + Error::Io(ref error) => fmt::Display::fmt(error, fmt), + Error::FromUtf8(ref error) => fmt::Display::fmt(error, fmt), + } + } +} + +impl From for Error { + fn from(error: io::Error) -> Error { + Error::Io(error) + } +} + +impl From for Error { + fn from(error: FromUtf8Error) -> Error { + Error::FromUtf8(error) + } +} + +impl de::Error for Error { + fn custom(msg: T) -> Error { + Error::Syntax(ErrorCode::Custom(msg.to_string()), 0, 0) + } +} + +impl ser::Error for Error { + /// Raised when there is general error when deserializing a type. + fn custom(msg: T) -> Error { + Error::Syntax(ErrorCode::Custom(msg.to_string()), 0, 0) + } +} + +/// Helper alias for `Result` objects that return a JSON `Error`. +pub type Result = result::Result; diff --git a/crates/nu-json/src/lib.rs b/crates/nu-json/src/lib.rs new file mode 100644 index 0000000000..6a196c5273 --- /dev/null +++ b/crates/nu-json/src/lib.rs @@ -0,0 +1,13 @@ +pub use self::de::{ + from_iter, from_reader, from_slice, from_str, Deserializer, StreamDeserializer, +}; +pub use self::error::{Error, ErrorCode, Result}; +pub use self::ser::{to_string, to_vec, to_writer, Serializer}; +pub use self::value::{from_value, to_value, Map, Value}; + +pub mod builder; +pub mod de; +pub mod error; +pub mod ser; +mod util; +pub mod value; diff --git a/crates/nu-json/src/ser.rs b/crates/nu-json/src/ser.rs new file mode 100644 index 0000000000..01ea63e862 --- /dev/null +++ b/crates/nu-json/src/ser.rs @@ -0,0 +1,1020 @@ +//! Hjson Serialization +//! +//! This module provides for Hjson serialization with the type `Serializer`. + +use std::fmt::{Display, LowerExp}; +use std::io; +use std::num::FpCategory; + +use super::error::{Error, ErrorCode, Result}; +use serde::ser; + +use super::util::ParseNumber; + +use regex::Regex; + +use lazy_static::lazy_static; + +/// A structure for serializing Rust values into Hjson. +pub struct Serializer { + writer: W, + formatter: F, +} + +impl<'a, W> Serializer> +where + W: io::Write, +{ + /// Creates a new Hjson serializer. + #[inline] + pub fn new(writer: W) -> Self { + Serializer::with_formatter(writer, HjsonFormatter::new()) + } +} + +impl Serializer +where + W: io::Write, + F: Formatter, +{ + /// Creates a new Hjson visitor whose output will be written to the writer + /// specified. + #[inline] + pub fn with_formatter(writer: W, formatter: F) -> Self { + Serializer { writer, formatter } + } + + /// Unwrap the `Writer` from the `Serializer`. + #[inline] + pub fn into_inner(self) -> W { + self.writer + } +} + +#[doc(hidden)] +#[derive(Eq, PartialEq)] +pub enum State { + Empty, + First, + Rest, +} + +#[doc(hidden)] +pub struct Compound<'a, W, F> { + ser: &'a mut Serializer, + state: State, +} + +impl<'a, W, F> ser::Serializer for &'a mut Serializer +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + type SerializeSeq = Compound<'a, W, F>; + type SerializeTuple = Compound<'a, W, F>; + type SerializeTupleStruct = Compound<'a, W, F>; + type SerializeTupleVariant = Compound<'a, W, F>; + type SerializeMap = Compound<'a, W, F>; + type SerializeStruct = Compound<'a, W, F>; + type SerializeStructVariant = Compound<'a, W, F>; + + #[inline] + fn serialize_bool(self, value: bool) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + if value { + self.writer.write_all(b"true").map_err(From::from) + } else { + self.writer.write_all(b"false").map_err(From::from) + } + } + + #[inline] + fn serialize_i8(self, value: i8) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_i16(self, value: i16) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_i32(self, value: i32) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_i64(self, value: i64) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_u8(self, value: u8) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_u16(self, value: u16) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_u32(self, value: u32) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_u64(self, value: u64) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + write!(&mut self.writer, "{}", value).map_err(From::from) + } + + #[inline] + fn serialize_f32(self, value: f32) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + fmt_f32_or_null(&mut self.writer, if value == -0f32 { 0f32 } else { value }) + .map_err(From::from) + } + + #[inline] + fn serialize_f64(self, value: f64) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + fmt_f64_or_null(&mut self.writer, if value == -0f64 { 0f64 } else { value }) + .map_err(From::from) + } + + #[inline] + fn serialize_char(self, value: char) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + escape_char(&mut self.writer, value).map_err(From::from) + } + + #[inline] + fn serialize_str(self, value: &str) -> Result<()> { + quote_str(&mut self.writer, &mut self.formatter, value).map_err(From::from) + } + + #[inline] + fn serialize_bytes(self, value: &[u8]) -> Result<()> { + let mut seq = self.serialize_seq(Some(value.len()))?; + for byte in value { + ser::SerializeSeq::serialize_element(&mut seq, byte)? + } + ser::SerializeSeq::end(seq) + } + + #[inline] + fn serialize_unit(self) -> Result<()> { + self.formatter.start_value(&mut self.writer)?; + self.writer.write_all(b"null").map_err(From::from) + } + + #[inline] + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + self.serialize_unit() + } + + #[inline] + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + self.serialize_str(variant) + } + + /// Serialize newtypes without an object wrapper. + #[inline] + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + value.serialize(self) + } + + #[inline] + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + self.formatter.open(&mut self.writer, b'{')?; + self.formatter.comma(&mut self.writer, true)?; + escape_key(&mut self.writer, variant)?; + self.formatter.colon(&mut self.writer)?; + value.serialize(&mut *self)?; + self.formatter.close(&mut self.writer, b'}') + } + + #[inline] + fn serialize_none(self) -> Result<()> { + self.serialize_unit() + } + + #[inline] + fn serialize_some(self, value: &V) -> Result<()> + where + V: ?Sized + ser::Serialize, + { + value.serialize(self) + } + + #[inline] + fn serialize_seq(self, len: Option) -> Result { + let state = if len == Some(0) { + self.formatter.start_value(&mut self.writer)?; + self.writer.write_all(b"[]")?; + State::Empty + } else { + self.formatter.open(&mut self.writer, b'[')?; + State::First + }; + Ok(Compound { ser: self, state }) + } + + #[inline] + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_seq(Some(len)) + } + + #[inline] + fn serialize_tuple_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.serialize_seq(Some(len)) + } + + #[inline] + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.formatter.open(&mut self.writer, b'{')?; + self.formatter.comma(&mut self.writer, true)?; + escape_key(&mut self.writer, variant)?; + self.formatter.colon(&mut self.writer)?; + self.serialize_seq(Some(len)) + } + + #[inline] + fn serialize_map(self, len: Option) -> Result { + let state = if len == Some(0) { + self.formatter.start_value(&mut self.writer)?; + self.writer.write_all(b"{}")?; + State::Empty + } else { + self.formatter.open(&mut self.writer, b'{')?; + State::First + }; + Ok(Compound { ser: self, state }) + } + + #[inline] + fn serialize_struct(self, _name: &'static str, len: usize) -> Result { + self.serialize_map(Some(len)) + } + + #[inline] + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.formatter.open(&mut self.writer, b'{')?; + self.formatter.comma(&mut self.writer, true)?; + escape_key(&mut self.writer, variant)?; + self.formatter.colon(&mut self.writer)?; + self.serialize_map(Some(len)) + } +} + +impl<'a, W, F> ser::SerializeSeq for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: serde::Serialize, + { + self.ser + .formatter + .comma(&mut self.ser.writer, self.state == State::First)?; + self.state = State::Rest; + value.serialize(&mut *self.ser) + } + + fn end(self) -> Result { + match self.state { + State::Empty => Ok(()), + _ => self.ser.formatter.close(&mut self.ser.writer, b']'), + } + } +} + +impl<'a, W, F> ser::SerializeTuple for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: serde::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } +} + +impl<'a, W, F> ser::SerializeTupleStruct for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: serde::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } +} + +impl<'a, W, F> ser::SerializeTupleVariant for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: serde::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result { + match self.state { + State::Empty => {} + _ => self.ser.formatter.close(&mut self.ser.writer, b']')?, + } + self.ser.formatter.close(&mut self.ser.writer, b'}') + } +} + +impl<'a, W, F> ser::SerializeMap for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: serde::Serialize, + { + self.ser + .formatter + .comma(&mut self.ser.writer, self.state == State::First)?; + self.state = State::Rest; + + key.serialize(MapKeySerializer { ser: self.ser })?; + + self.ser.formatter.colon(&mut self.ser.writer) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: serde::Serialize, + { + value.serialize(&mut *self.ser) + } + + fn end(self) -> Result { + match self.state { + State::Empty => Ok(()), + _ => self.ser.formatter.close(&mut self.ser.writer, b'}'), + } + } +} + +impl<'a, W, F> ser::SerializeStruct for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: serde::Serialize, + { + ser::SerializeMap::serialize_entry(self, key, value) + } + + fn end(self) -> Result { + ser::SerializeMap::end(self) + } +} + +impl<'a, W, F> ser::SerializeStructVariant for Compound<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: serde::Serialize, + { + ser::SerializeStruct::serialize_field(self, key, value) + } + + fn end(self) -> Result { + match self.state { + State::Empty => {} + _ => self.ser.formatter.close(&mut self.ser.writer, b'}')?, + } + self.ser.formatter.close(&mut self.ser.writer, b'}') + } +} + +struct MapKeySerializer<'a, W: 'a, F: 'a> { + ser: &'a mut Serializer, +} + +impl<'a, W, F> ser::Serializer for MapKeySerializer<'a, W, F> +where + W: io::Write, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_str(self, value: &str) -> Result<()> { + escape_key(&mut self.ser.writer, value).map_err(From::from) + } + + type SerializeSeq = ser::Impossible<(), Error>; + type SerializeTuple = ser::Impossible<(), Error>; + type SerializeTupleStruct = ser::Impossible<(), Error>; + type SerializeTupleVariant = ser::Impossible<(), Error>; + type SerializeMap = ser::Impossible<(), Error>; + type SerializeStruct = ser::Impossible<(), Error>; + type SerializeStructVariant = ser::Impossible<(), Error>; + + fn serialize_bool(self, _value: bool) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_i8(self, _value: i8) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_i16(self, _value: i16) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_i32(self, _value: i32) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_i64(self, _value: i64) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_u8(self, _value: u8) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_u16(self, _value: u16) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_u32(self, _value: u32) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_u64(self, _value: u64) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_f32(self, _value: f32) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_f64(self, _value: f64) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_char(self, _value: char) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_bytes(self, _value: &[u8]) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_unit(self) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + ) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_newtype_struct(self, _name: &'static str, _value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_none(self) -> Result<()> { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_some(self, _value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_seq(self, _len: Option) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_tuple(self, _len: usize) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_map(self, _len: Option) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)) + } +} + +/// This trait abstracts away serializing the JSON control characters +pub trait Formatter { + /// Called when serializing a '{' or '['. + fn open(&mut self, writer: &mut W, ch: u8) -> Result<()> + where + W: io::Write; + + /// Called when serializing a ','. + fn comma(&mut self, writer: &mut W, first: bool) -> Result<()> + where + W: io::Write; + + /// Called when serializing a ':'. + fn colon(&mut self, writer: &mut W) -> Result<()> + where + W: io::Write; + + /// Called when serializing a '}' or ']'. + fn close(&mut self, writer: &mut W, ch: u8) -> Result<()> + where + W: io::Write; + + /// Newline with indent. + fn newline(&mut self, writer: &mut W, add_indent: i32) -> Result<()> + where + W: io::Write; + + /// Start a value. + fn start_value(&mut self, writer: &mut W) -> Result<()> + where + W: io::Write; +} + +struct HjsonFormatter<'a> { + current_indent: usize, + current_is_array: bool, + stack: Vec, + at_colon: bool, + indent: &'a [u8], + braces_same_line: bool, +} + +impl<'a> HjsonFormatter<'a> { + /// Construct a formatter that defaults to using two spaces for indentation. + pub fn new() -> Self { + HjsonFormatter::with_indent(b" ") + } + + /// Construct a formatter that uses the `indent` string for indentation. + pub fn with_indent(indent: &'a [u8]) -> Self { + HjsonFormatter { + current_indent: 0, + current_is_array: false, + stack: Vec::new(), + at_colon: false, + indent, + braces_same_line: false, + } + } +} + +impl<'a> Formatter for HjsonFormatter<'a> { + fn open(&mut self, writer: &mut W, ch: u8) -> Result<()> + where + W: io::Write, + { + if self.current_indent > 0 && !self.current_is_array && !self.braces_same_line { + self.newline(writer, 0)?; + } else { + self.start_value(writer)?; + } + self.current_indent += 1; + self.stack.push(self.current_is_array); + self.current_is_array = ch == b'['; + writer.write_all(&[ch]).map_err(From::from) + } + + fn comma(&mut self, writer: &mut W, _: bool) -> Result<()> + where + W: io::Write, + { + writer.write_all(b"\n")?; + indent(writer, self.current_indent, self.indent) + } + + fn colon(&mut self, writer: &mut W) -> Result<()> + where + W: io::Write, + { + self.at_colon = !self.braces_same_line; + writer + .write_all(if self.braces_same_line { b": " } else { b":" }) + .map_err(From::from) + } + + fn close(&mut self, writer: &mut W, ch: u8) -> Result<()> + where + W: io::Write, + { + self.current_indent -= 1; + self.current_is_array = self.stack.pop().expect("Internal error: json parsing"); + writer.write_all(b"\n")?; + indent(writer, self.current_indent, self.indent)?; + writer.write_all(&[ch]).map_err(From::from) + } + + fn newline(&mut self, writer: &mut W, add_indent: i32) -> Result<()> + where + W: io::Write, + { + self.at_colon = false; + writer.write_all(b"\n")?; + let ii = self.current_indent as i32 + add_indent; + indent(writer, if ii < 0 { 0 } else { ii as usize }, self.indent) + } + + fn start_value(&mut self, writer: &mut W) -> Result<()> + where + W: io::Write, + { + if self.at_colon { + self.at_colon = false; + writer.write_all(b" ")? + } + Ok(()) + } +} + +/// Serializes and escapes a `&[u8]` into a Hjson string. +#[inline] +pub fn escape_bytes(wr: &mut W, bytes: &[u8]) -> Result<()> +where + W: io::Write, +{ + wr.write_all(b"\"")?; + + let mut start = 0; + + for (i, byte) in bytes.iter().enumerate() { + let escaped = match *byte { + b'"' => b"\\\"", + b'\\' => b"\\\\", + b'\x08' => b"\\b", + b'\x0c' => b"\\f", + b'\n' => b"\\n", + b'\r' => b"\\r", + b'\t' => b"\\t", + _ => { + continue; + } + }; + + if start < i { + wr.write_all(&bytes[start..i])?; + } + + wr.write_all(escaped)?; + + start = i + 1; + } + + if start != bytes.len() { + wr.write_all(&bytes[start..])?; + } + + wr.write_all(b"\"")?; + Ok(()) +} + +/// Serializes and escapes a `&str` into a Hjson string. +#[inline] +pub fn quote_str(wr: &mut W, formatter: &mut F, value: &str) -> Result<()> +where + W: io::Write, + F: Formatter, +{ + lazy_static! { + // NEEDS_ESCAPE tests if the string can be written without escapes + static ref NEEDS_ESCAPE: Regex = Regex::new("[\\\\\"\x00-\x1f\x7f-\u{9f}\u{00ad}\u{0600}-\u{0604}\u{070f}\u{17b4}\u{17b5}\u{200c}-\u{200f}\u{2028}-\u{202f}\u{2060}-\u{206f}\u{feff}\u{fff0}-\u{ffff}]").expect("Internal error: json parsing"); + // NEEDS_QUOTES tests if the string can be written as a quoteless string (includes needsEscape but without \\ and \") + static ref NEEDS_QUOTES: Regex = Regex::new("^\\s|^\"|^'''|^#|^/\\*|^//|^\\{|^\\}|^\\[|^\\]|^:|^,|\\s$|[\x00-\x1f\x7f-\u{9f}\u{00ad}\u{0600}-\u{0604}\u{070f}\u{17b4}\u{17b5}\u{200c}-\u{200f}\u{2028}-\u{202f}\u{2060}-\u{206f}\u{feff}\u{fff0}-\u{ffff}]").expect("Internal error: json parsing"); + // NEEDS_ESCAPEML tests if the string can be written as a multiline string (includes needsEscape but without \n, \r, \\ and \") + static ref NEEDS_ESCAPEML: Regex = Regex::new("'''|[\x00-\x09\x0b\x0c\x0e-\x1f\x7f-\u{9f}\u{00ad}\u{0600}-\u{0604}\u{070f}\u{17b4}\u{17b5}\u{200c}-\u{200f}\u{2028}-\u{202f}\u{2060}-\u{206f}\u{feff}\u{fff0}-\u{ffff}]").expect("Internal error: json parsing"); + // starts with a keyword and optionally is followed by a comment + static ref STARTS_WITH_KEYWORD: Regex = Regex::new(r#"^(true|false|null)\s*((,|\]|\}|#|//|/\*).*)?$"#).expect("Internal error: json parsing"); + } + + if value.is_empty() { + formatter.start_value(wr)?; + return escape_bytes(wr, value.as_bytes()); + } + + // Check if we can insert this string without quotes + // see hjson syntax (must not parse as true, false, null or number) + + let mut pn = ParseNumber::new(value.bytes()); + let is_number = pn.parse(true).is_ok(); + + if is_number || NEEDS_QUOTES.is_match(value) || STARTS_WITH_KEYWORD.is_match(value) { + // First check if the string can be expressed in multiline format or + // we must replace the offending characters with safe escape sequences. + + if NEEDS_ESCAPE.is_match(value) && !NEEDS_ESCAPEML.is_match(value) + /* && !isRootObject */ + { + ml_str(wr, formatter, value) + } else { + formatter.start_value(wr)?; + escape_bytes(wr, value.as_bytes()) + } + } else { + // without quotes + formatter.start_value(wr)?; + wr.write_all(value.as_bytes()).map_err(From::from) + } +} + +/// Serializes and escapes a `&str` into a multiline Hjson string. +pub fn ml_str(wr: &mut W, formatter: &mut F, value: &str) -> Result<()> +where + W: io::Write, + F: Formatter, +{ + // wrap the string into the ''' (multiline) format + + let a: Vec<&str> = value.split('\n').collect(); + + if a.len() == 1 { + // The string contains only a single line. We still use the multiline + // format as it avoids escaping the \ character (e.g. when used in a + // regex). + formatter.start_value(wr)?; + wr.write_all(b"'''")?; + wr.write_all(a[0].as_bytes())?; + wr.write_all(b"'''")? + } else { + formatter.newline(wr, 1)?; + wr.write_all(b"'''")?; + for line in a { + formatter.newline(wr, if !line.is_empty() { 1 } else { -999 })?; + wr.write_all(line.as_bytes())?; + } + formatter.newline(wr, 1)?; + wr.write_all(b"'''")?; + } + Ok(()) +} + +/// Serializes and escapes a `&str` into a Hjson key. +#[inline] +pub fn escape_key(wr: &mut W, value: &str) -> Result<()> +where + W: io::Write, +{ + lazy_static! { + static ref NEEDS_ESCAPE_NAME: Regex = + Regex::new(r#"[,\{\[\}\]\s:#"]|//|/\*|'''|^$"#).expect("Internal error: json parsing"); + } + + // Check if we can insert this name without quotes + if NEEDS_ESCAPE_NAME.is_match(value) { + escape_bytes(wr, value.as_bytes()).map_err(From::from) + } else { + wr.write_all(value.as_bytes()).map_err(From::from) + } +} + +#[inline] +fn escape_char(wr: &mut W, value: char) -> Result<()> +where + W: io::Write, +{ + // FIXME: this allocation is required in order to be compatible with stable + // rust, which doesn't support encoding a `char` into a stack buffer. + let mut s = String::new(); + s.push(value); + escape_bytes(wr, s.as_bytes()) +} + +fn fmt_f32_or_null(wr: &mut W, value: f32) -> Result<()> +where + W: io::Write, +{ + match value.classify() { + FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null")?, + _ => wr.write_all(fmt_small(value).as_bytes())?, + } + + Ok(()) +} + +fn fmt_f64_or_null(wr: &mut W, value: f64) -> Result<()> +where + W: io::Write, +{ + match value.classify() { + FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null")?, + _ => wr.write_all(fmt_small(value).as_bytes())?, + } + + Ok(()) +} + +fn indent(wr: &mut W, n: usize, s: &[u8]) -> Result<()> +where + W: io::Write, +{ + for _ in 0..n { + wr.write_all(s)?; + } + + Ok(()) +} + +// format similar to es6 +fn fmt_small(value: N) -> String +where + N: Display + LowerExp, +{ + let f1 = value.to_string(); + let f2 = format!("{:e}", value); + if f1.len() <= f2.len() + 1 { + f1 + } else if !f2.contains("e-") { + f2.replace("e", "e+") + } else { + f2 + } +} + +/// Encode the specified struct into a Hjson `[u8]` writer. +#[inline] +pub fn to_writer(writer: &mut W, value: &T) -> Result<()> +where + W: io::Write, + T: ser::Serialize, +{ + let mut ser = Serializer::new(writer); + value.serialize(&mut ser)?; + Ok(()) +} + +/// Encode the specified struct into a Hjson `[u8]` buffer. +#[inline] +pub fn to_vec(value: &T) -> Result> +where + T: ser::Serialize, +{ + // We are writing to a Vec, which doesn't fail. So we can ignore + // the error. + let mut writer = Vec::with_capacity(128); + to_writer(&mut writer, value)?; + Ok(writer) +} + +/// Encode the specified struct into a Hjson `String` buffer. +#[inline] +pub fn to_string(value: &T) -> Result +where + T: ser::Serialize, +{ + let vec = to_vec(value)?; + let string = String::from_utf8(vec)?; + Ok(string) +} diff --git a/crates/nu-json/src/util.rs b/crates/nu-json/src/util.rs new file mode 100644 index 0000000000..5f650b1095 --- /dev/null +++ b/crates/nu-json/src/util.rs @@ -0,0 +1,333 @@ +use std::io; +use std::str; + +use super::error::{Error, ErrorCode, Result}; + +pub struct StringReader> { + iter: Iter, + line: usize, + col: usize, + ch: Vec, +} + +impl StringReader +where + Iter: Iterator, +{ + #[inline] + pub fn new(iter: Iter) -> Self { + StringReader { + iter, + line: 1, + col: 0, + ch: Vec::new(), + } + } + + fn next(&mut self) -> Option> { + match self.iter.next() { + None => None, + Some(b'\n') => { + self.line += 1; + self.col = 0; + Some(Ok(b'\n')) + } + Some(c) => { + self.col += 1; + Some(Ok(c)) + } + } + } + + pub fn pos(&mut self) -> (usize, usize) { + (self.line, self.col) + } + + pub fn eof(&mut self) -> Result { + Ok(self.peek()?.is_none()) + } + + pub fn peek_next(&mut self, idx: usize) -> Result> { + while self.ch.len() <= idx { + match self.next() { + Some(Err(err)) => return Err(Error::Io(err)), + Some(Ok(ch)) => self.ch.push(ch), + None => return Ok(None), + } + } + Ok(Some(self.ch[idx])) + } + + // pub fn peek_next_or_null(&mut self, idx: usize) -> Result { + // Ok(try!(self.peek_next(idx)).unwrap_or(b'\x00')) + // } + + pub fn peek(&mut self) -> Result> { + self.peek_next(0) + } + + pub fn peek_or_null(&mut self) -> Result { + Ok(self.peek()?.unwrap_or(b'\x00')) + } + + pub fn eat_char(&mut self) -> u8 { + self.ch.remove(0) + } + + pub fn uneat_char(&mut self, ch: u8) { + self.ch.insert(0, ch); + } + + pub fn next_char(&mut self) -> Result> { + match self.ch.first() { + Some(&ch) => { + self.eat_char(); + Ok(Some(ch)) + } + None => match self.next() { + Some(Err(err)) => Err(Error::Io(err)), + Some(Ok(ch)) => Ok(Some(ch)), + None => Ok(None), + }, + } + } + + pub fn next_char_or_null(&mut self) -> Result { + Ok(self.next_char()?.unwrap_or(b'\x00')) + } + + fn eat_line(&mut self) -> Result<()> { + loop { + match self.peek()? { + Some(b'\n') | None => return Ok(()), + _ => {} + } + self.eat_char(); + } + } + + pub fn parse_whitespace(&mut self) -> Result<()> { + loop { + match self.peek_or_null()? { + b' ' | b'\n' | b'\t' | b'\r' => { + self.eat_char(); + } + b'#' => self.eat_line()?, + b'/' => { + match self.peek_next(1)? { + Some(b'/') => self.eat_line()?, + Some(b'*') => { + self.eat_char(); + self.eat_char(); + while !(self.peek()?.unwrap_or(b'*') == b'*' + && self.peek_next(1)?.unwrap_or(b'/') == b'/') + { + self.eat_char(); + } + self.eat_char(); + self.eat_char(); + } + Some(_) => { + self.eat_char(); + } + None => return Err(self.error(ErrorCode::TrailingCharacters)), //todo + } + } + _ => { + return Ok(()); + } + } + } + } + + pub fn error(&mut self, reason: ErrorCode) -> Error { + Error::Syntax(reason, self.line, self.col) + } +} + +pub enum Number { + I64(i64), + U64(u64), + F64(f64), +} + +pub struct ParseNumber> { + rdr: StringReader, + result: Vec, +} + +// macro_rules! try_or_invalid { +// ($e:expr) => { +// match $e { +// Some(v) => v, +// None => { return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); } +// } +// } +// } + +impl> ParseNumber { + #[inline] + pub fn new(iter: Iter) -> Self { + ParseNumber { + rdr: StringReader::new(iter), + result: Vec::new(), + } + } + + pub fn parse(&mut self, stop_at_next: bool) -> Result { + match self.try_parse() { + Ok(()) => { + self.rdr.parse_whitespace()?; + + let mut ch = self.rdr.next_char_or_null()?; + + if stop_at_next { + let ch2 = self.rdr.peek_or_null()?; + // end scan if we find a punctuator character like ,}] or a comment + if ch == b',' + || ch == b'}' + || ch == b']' + || ch == b'#' + || ch == b'/' && (ch2 == b'/' || ch2 == b'*') + { + ch = b'\x00'; + } + } + + match ch { + b'\x00' => { + let res = + str::from_utf8(&self.result).expect("Internal error: json parsing"); + + let mut is_float = false; + for ch in res.chars() { + if ch == '.' || ch == 'e' || ch == 'E' { + is_float = true; + break; + } + } + + if is_float { + Ok(Number::F64( + res.parse::().expect("Internal error: json parsing"), + )) + } else if res.starts_with('-') { + Ok(Number::I64( + res.parse::().expect("Internal error: json parsing"), + )) + } else { + Ok(Number::U64( + res.parse::().expect("Internal error: json parsing"), + )) + } + } + _ => Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)), + } + } + Err(e) => Err(e), + } + } + + fn try_parse(&mut self) -> Result<()> { + if self.rdr.peek_or_null()? == b'-' { + self.result.push(self.rdr.eat_char()); + } + + let mut has_value = false; + + if self.rdr.peek_or_null()? == b'0' { + self.result.push(self.rdr.eat_char()); + has_value = true; + + // There can be only one leading '0'. + if let b'0'..=b'9' = self.rdr.peek_or_null()? { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + } + + loop { + match self.rdr.peek_or_null()? { + b'0'..=b'9' => { + self.result.push(self.rdr.eat_char()); + has_value = true; + } + b'.' => { + if !has_value { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + self.rdr.eat_char(); + return self.try_decimal(); + } + b'e' | b'E' => { + if !has_value { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + self.rdr.eat_char(); + return self.try_exponent(); + } + _ => { + if !has_value { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + return Ok(()); + } + } + } + } + + fn try_decimal(&mut self) -> Result<()> { + self.result.push(b'.'); + + // Make sure a digit follows the decimal place. + match self.rdr.next_char_or_null()? { + c @ b'0'..=b'9' => { + self.result.push(c); + } + _ => { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + }; + + while let b'0'..=b'9' = self.rdr.peek_or_null()? { + self.result.push(self.rdr.eat_char()); + } + + match self.rdr.peek_or_null()? { + b'e' | b'E' => { + self.rdr.eat_char(); + self.try_exponent() + } + _ => Ok(()), + } + } + + fn try_exponent(&mut self) -> Result<()> { + self.result.push(b'e'); + + match self.rdr.peek_or_null()? { + b'+' => { + self.result.push(self.rdr.eat_char()); + } + b'-' => { + self.result.push(self.rdr.eat_char()); + } + _ => {} + }; + + // Make sure a digit follows the exponent place. + match self.rdr.next_char_or_null()? { + c @ b'0'..=b'9' => { + self.result.push(c); + } + _ => { + return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); + } + }; + + while let b'0'..=b'9' = self.rdr.peek_or_null()? { + self.result.push(self.rdr.eat_char()); + } + + Ok(()) + } +} diff --git a/crates/nu-json/src/value.rs b/crates/nu-json/src/value.rs new file mode 100644 index 0000000000..0d3533f9c8 --- /dev/null +++ b/crates/nu-json/src/value.rs @@ -0,0 +1,1158 @@ +#[cfg(not(feature = "preserve_order"))] +use std::collections::{btree_map, BTreeMap}; + +#[cfg(feature = "preserve_order")] +use linked_hash_map::{self, LinkedHashMap}; + +use std::fmt; +use std::io; +use std::str; +use std::vec; + +use num_traits::NumCast; + +use serde::de; +use serde::ser; + +use crate::error::{Error, ErrorCode}; + +type Result = std::result::Result; + +/// Represents a key/value type. +#[cfg(not(feature = "preserve_order"))] +pub type Map = BTreeMap; +/// Represents a key/value type. +#[cfg(feature = "preserve_order")] +pub type Map = LinkedHashMap; + +/// Represents the `IntoIter` type. +#[cfg(not(feature = "preserve_order"))] +pub type MapIntoIter = btree_map::IntoIter; +/// Represents the IntoIter type. +#[cfg(feature = "preserve_order")] +pub type MapIntoIter = linked_hash_map::IntoIter; + +fn map_with_capacity(size: Option) -> Map { + #[cfg(not(feature = "preserve_order"))] + { + let _ = size; + BTreeMap::new() + } + + #[cfg(feature = "preserve_order")] + { + LinkedHashMap::with_capacity(size.unwrap_or(0)) + } +} + +/// Represents a Hjson/JSON value +#[derive(Clone, PartialEq)] +pub enum Value { + /// Represents a JSON null value + Null, + + /// Represents a JSON Boolean + Bool(bool), + + /// Represents a JSON signed integer + I64(i64), + + /// Represents a JSON unsigned integer + U64(u64), + + /// Represents a JSON floating point number + F64(f64), + + /// Represents a JSON string + String(String), + + /// Represents a JSON array + Array(Vec), + + /// Represents a JSON object + Object(Map), +} + +impl Value { + /// If the `Value` is an Object, returns the value associated with the provided key. + /// Otherwise, returns None. + pub fn find<'a>(&'a self, key: &str) -> Option<&'a Value> { + match *self { + Value::Object(ref map) => map.get(key), + _ => None, + } + } + + /// Attempts to get a nested Value Object for each key in `keys`. + /// If any key is found not to exist, find_path will return None. + /// Otherwise, it will return the `Value` associated with the final key. + pub fn find_path<'a>(&'a self, keys: &[&str]) -> Option<&'a Value> { + let mut target = self; + for key in keys { + match target.find(key) { + Some(t) => { + target = t; + } + None => return None, + } + } + Some(target) + } + + /// Looks up a value by a JSON Pointer. + /// + /// JSON Pointer defines a string syntax for identifying a specific value + /// within a JavaScript Object Notation (JSON) document. + /// + /// A Pointer is a Unicode string with the reference tokens separated by `/`. + /// Inside tokens `/` is replaced by `~1` and `~` is replaced by `~0`. The + /// addressed value is returned and if there is no such value `None` is + /// returned. + /// + /// For more information read [RFC6901](https://tools.ietf.org/html/rfc6901). + pub fn pointer<'a>(&'a self, pointer: &str) -> Option<&'a Value> { + fn parse_index(s: &str) -> Option { + if s.starts_with('+') || (s.starts_with('0') && s.len() != 1) { + return None; + } + s.parse().ok() + } + if pointer.is_empty() { + return Some(self); + } + if !pointer.starts_with('/') { + return None; + } + let mut target = self; + for escaped_token in pointer.split('/').skip(1) { + let token = escaped_token.replace("~1", "/").replace("~0", "~"); + let target_opt = match *target { + Value::Object(ref map) => map.get(&token[..]), + Value::Array(ref list) => parse_index(&token[..]).and_then(|x| list.get(x)), + _ => return None, + }; + if let Some(t) = target_opt { + target = t; + } else { + return None; + } + } + Some(target) + } + + /// If the `Value` is an Object, performs a depth-first search until + /// a value associated with the provided key is found. If no value is found + /// or the `Value` is not an Object, returns None. + pub fn search<'a>(&'a self, key: &str) -> Option<&'a Value> { + match self { + Value::Object(map) => map + .get(key) + .or_else(|| map.values().find_map(|v| v.search(key))), + _ => None, + } + } + + /// Returns true if the `Value` is an Object. Returns false otherwise. + pub fn is_object(&self) -> bool { + self.as_object().is_some() + } + + /// If the `Value` is an Object, returns the associated Map. + /// Returns None otherwise. + pub fn as_object(&self) -> Option<&Map> { + match *self { + Value::Object(ref map) => Some(map), + _ => None, + } + } + + /// If the `Value` is an Object, returns the associated mutable Map. + /// Returns None otherwise. + pub fn as_object_mut(&mut self) -> Option<&mut Map> { + match *self { + Value::Object(ref mut map) => Some(map), + _ => None, + } + } + + /// Returns true if the `Value` is an Array. Returns false otherwise. + pub fn is_array(&self) -> bool { + self.as_array().is_some() + } + + /// If the `Value` is an Array, returns the associated vector. + /// Returns None otherwise. + pub fn as_array(&self) -> Option<&Vec> { + match self { + Value::Array(array) => Some(array), + _ => None, + } + } + + /// If the `Value` is an Array, returns the associated mutable vector. + /// Returns None otherwise. + pub fn as_array_mut(&mut self) -> Option<&mut Vec> { + match self { + Value::Array(list) => Some(list), + _ => None, + } + } + + /// Returns true if the `Value` is a String. Returns false otherwise. + pub fn is_string(&self) -> bool { + self.as_str().is_some() + } + + /// If the `Value` is a String, returns the associated str. + /// Returns None otherwise. + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(s) => Some(s), + _ => None, + } + } + + /// Returns true if the `Value` is a Number. Returns false otherwise. + pub fn is_number(&self) -> bool { + matches!(self, Value::I64(_) | Value::U64(_) | Value::F64(_)) + } + + /// Returns true if the `Value` is a i64. Returns false otherwise. + pub fn is_i64(&self) -> bool { + matches!(self, Value::I64(_)) + } + + /// Returns true if the `Value` is a u64. Returns false otherwise. + pub fn is_u64(&self) -> bool { + matches!(self, Value::U64(_)) + } + + /// Returns true if the `Value` is a f64. Returns false otherwise. + pub fn is_f64(&self) -> bool { + matches!(self, Value::F64(_)) + } + + /// If the `Value` is a number, return or cast it to a i64. + /// Returns None otherwise. + pub fn as_i64(&self) -> Option { + match *self { + Value::I64(n) => Some(n), + Value::U64(n) => NumCast::from(n), + _ => None, + } + } + + /// If the `Value` is a number, return or cast it to a u64. + /// Returns None otherwise. + pub fn as_u64(&self) -> Option { + match *self { + Value::I64(n) => NumCast::from(n), + Value::U64(n) => Some(n), + _ => None, + } + } + + /// If the `Value` is a number, return or cast it to a f64. + /// Returns None otherwise. + pub fn as_f64(&self) -> Option { + match *self { + Value::I64(n) => NumCast::from(n), + Value::U64(n) => NumCast::from(n), + Value::F64(n) => Some(n), + _ => None, + } + } + + /// Returns true if the `Value` is a Boolean. Returns false otherwise. + pub fn is_boolean(&self) -> bool { + self.as_bool().is_some() + } + + /// If the `Value` is a Boolean, returns the associated bool. + /// Returns None otherwise. + pub fn as_bool(&self) -> Option { + match *self { + Value::Bool(b) => Some(b), + _ => None, + } + } + + /// Returns true if the `Value` is a Null. Returns false otherwise. + pub fn is_null(&self) -> bool { + self.as_null().is_some() + } + + /// If the `Value` is a Null, returns (). + /// Returns None otherwise. + pub fn as_null(&self) -> Option<()> { + match self { + Value::Null => Some(()), + _ => None, + } + } + + fn as_unexpected(&self) -> de::Unexpected<'_> { + match *self { + Value::Null => de::Unexpected::Unit, + Value::Bool(v) => de::Unexpected::Bool(v), + Value::I64(v) => de::Unexpected::Signed(v), + Value::U64(v) => de::Unexpected::Unsigned(v), + Value::F64(v) => de::Unexpected::Float(v), + Value::String(ref v) => de::Unexpected::Str(v), + Value::Array(_) => de::Unexpected::Seq, + Value::Object(_) => de::Unexpected::Map, + } + } +} + +impl ser::Serialize for Value { + #[inline] + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + match *self { + Value::Null => serializer.serialize_unit(), + Value::Bool(v) => serializer.serialize_bool(v), + Value::I64(v) => serializer.serialize_i64(v), + Value::U64(v) => serializer.serialize_u64(v), + Value::F64(v) => serializer.serialize_f64(v), + Value::String(ref v) => serializer.serialize_str(v), + Value::Array(ref v) => v.serialize(serializer), + Value::Object(ref v) => v.serialize(serializer), + } + } +} + +impl<'de> de::Deserialize<'de> for Value { + #[inline] + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + struct ValueVisitor; + + impl<'de> de::Visitor<'de> for ValueVisitor { + type Value = Value; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a json value") + } + + #[inline] + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + #[inline] + fn visit_i64(self, value: i64) -> Result { + if value < 0 { + Ok(Value::I64(value)) + } else { + Ok(Value::U64(value as u64)) + } + } + + #[inline] + fn visit_u64(self, value: u64) -> Result { + Ok(Value::U64(value)) + } + + #[inline] + fn visit_f64(self, value: f64) -> Result { + Ok(Value::F64(value)) + } + + #[inline] + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + self.visit_string(String::from(value)) + } + + #[inline] + fn visit_string(self, value: String) -> Result { + Ok(Value::String(value)) + } + + #[inline] + fn visit_none(self) -> Result { + Ok(Value::Null) + } + + #[inline] + fn visit_some(self, deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + de::Deserialize::deserialize(deserializer) + } + + #[inline] + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + #[inline] + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + let mut v = match seq.size_hint() { + Some(cap) => Vec::with_capacity(cap), + None => Vec::new(), + }; + + while let Some(el) = seq.next_element()? { + v.push(el) + } + + Ok(Value::Array(v)) + } + + #[inline] + fn visit_map(self, mut map: A) -> Result + where + A: de::MapAccess<'de>, + { + let mut values = map_with_capacity(map.size_hint()); + while let Some((k, v)) = map.next_entry()? { + values.insert(k, v); + } + Ok(Value::Object(values)) + } + } + + deserializer.deserialize_any(ValueVisitor) + } +} + +struct WriterFormatter<'a, 'b: 'a> { + inner: &'a mut fmt::Formatter<'b>, +} + +impl<'a, 'b> io::Write for WriterFormatter<'a, 'b> { + fn write(&mut self, buf: &[u8]) -> io::Result { + fn io_error(_: E) -> io::Error { + // Value does not matter because fmt::Debug and fmt::Display impls + // below just map it to fmt::Error + io::Error::new(io::ErrorKind::Other, "fmt error") + } + let s = str::from_utf8(buf).map_err(io_error)?; + self.inner.write_str(s).map_err(io_error)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl fmt::Debug for Value { + /// Serializes a Hjson value into a string + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut wr = WriterFormatter { inner: f }; + super::ser::to_writer(&mut wr, self).map_err(|_| fmt::Error) + } +} + +impl fmt::Display for Value { + /// Serializes a Hjson value into a string + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut wr = WriterFormatter { inner: f }; + super::ser::to_writer(&mut wr, self).map_err(|_| fmt::Error) + } +} + +impl str::FromStr for Value { + type Err = Error; + fn from_str(s: &str) -> Result { + super::de::from_str(s) + } +} + +/// Create a `serde::Serializer` that serializes a `Serialize`e into a `Value`. +#[derive(Default)] +pub struct Serializer; + +impl ser::Serializer for Serializer { + type Ok = Value; + type Error = Error; + + type SerializeSeq = SerializeVec; + type SerializeTuple = SerializeVec; + type SerializeTupleStruct = SerializeVec; + type SerializeTupleVariant = SerializeTupleVariant; + type SerializeMap = SerializeMap; + type SerializeStruct = SerializeMap; + type SerializeStructVariant = SerializeStructVariant; + + #[inline] + fn serialize_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + #[inline] + fn serialize_i8(self, value: i8) -> Result { + self.serialize_i64(value as i64) + } + + #[inline] + fn serialize_i16(self, value: i16) -> Result { + self.serialize_i64(value as i64) + } + + #[inline] + fn serialize_i32(self, value: i32) -> Result { + self.serialize_i64(value as i64) + } + + fn serialize_i64(self, value: i64) -> Result { + let v = if value < 0 { + Value::I64(value) + } else { + Value::U64(value as u64) + }; + Ok(v) + } + + #[inline] + fn serialize_u8(self, value: u8) -> Result { + self.serialize_u64(value as u64) + } + + #[inline] + fn serialize_u16(self, value: u16) -> Result { + self.serialize_u64(value as u64) + } + + #[inline] + fn serialize_u32(self, value: u32) -> Result { + self.serialize_u64(value as u64) + } + + #[inline] + fn serialize_u64(self, value: u64) -> Result { + Ok(Value::U64(value)) + } + + #[inline] + fn serialize_f32(self, value: f32) -> Result { + self.serialize_f64(value as f64) + } + + #[inline] + fn serialize_f64(self, value: f64) -> Result { + Ok(Value::F64(value)) + } + + #[inline] + fn serialize_char(self, value: char) -> Result { + let mut s = String::new(); + s.push(value); + self.serialize_str(&s) + } + + #[inline] + fn serialize_str(self, value: &str) -> Result { + Ok(Value::String(String::from(value))) + } + + fn serialize_bytes(self, value: &[u8]) -> Result { + let mut state = self.serialize_seq(Some(value.len()))?; + for byte in value { + ser::SerializeSeq::serialize_element(&mut state, byte)?; + } + ser::SerializeSeq::end(state) + } + + #[inline] + fn serialize_unit(self) -> Result { + Ok(Value::Null) + } + + #[inline] + fn serialize_unit_struct(self, _name: &'static str) -> Result { + self.serialize_unit() + } + + #[inline] + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result { + self.serialize_str(variant) + } + + #[inline] + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result + where + T: ?Sized + ser::Serialize, + { + value.serialize(self) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let mut values = Map::new(); + values.insert(String::from(variant), to_value(&value)?); + Ok(Value::Object(values)) + } + + #[inline] + fn serialize_none(self) -> Result { + self.serialize_unit() + } + + #[inline] + fn serialize_some(self, value: &V) -> Result + where + V: ?Sized + ser::Serialize, + { + value.serialize(self) + } + + #[inline] + fn serialize_seq(self, len: Option) -> Result { + Ok(SerializeVec { + vec: Vec::with_capacity(len.unwrap_or(0)), + }) + } + + #[inline] + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_seq(Some(len)) + } + + #[inline] + fn serialize_tuple_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.serialize_seq(Some(len)) + } + + #[inline] + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(SerializeTupleVariant { + name: variant, + vec: Vec::with_capacity(len), + }) + } + + #[inline] + fn serialize_map(self, len: Option) -> Result { + Ok(SerializeMap { + map: map_with_capacity(len), + next_key: None, + }) + } + + #[inline] + fn serialize_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.serialize_map(Some(len)) + } + + #[inline] + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(SerializeStructVariant { + name: variant, + map: map_with_capacity(Some(len)), + }) + } +} + +#[doc(hidden)] +pub struct SerializeVec { + vec: Vec, +} + +#[doc(hidden)] +pub struct SerializeTupleVariant { + name: &'static str, + vec: Vec, +} + +#[doc(hidden)] +pub struct SerializeMap { + map: Map, + next_key: Option, +} + +#[doc(hidden)] +pub struct SerializeStructVariant { + name: &'static str, + map: Map, +} + +impl ser::SerializeSeq for SerializeVec { + type Ok = Value; + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + self.vec.push(to_value(&value)?); + Ok(()) + } + + fn end(self) -> Result { + Ok(Value::Array(self.vec)) + } +} + +impl ser::SerializeTuple for SerializeVec { + type Ok = Value; + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } +} + +impl ser::SerializeTupleStruct for SerializeVec { + type Ok = Value; + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } +} + +impl ser::SerializeTupleVariant for SerializeTupleVariant { + type Ok = Value; + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + self.vec.push(to_value(&value)?); + Ok(()) + } + + fn end(self) -> Result { + let mut object = Map::new(); + + object.insert(self.name.to_owned(), Value::Array(self.vec)); + + Ok(Value::Object(object)) + } +} + +impl ser::SerializeMap for SerializeMap { + type Ok = Value; + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + match to_value(key)? { + Value::String(s) => self.next_key = Some(s), + _ => return Err(Error::Syntax(ErrorCode::KeyMustBeAString, 0, 0)), + }; + Ok(()) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + let key = self.next_key.take(); + // Panic because this indicates a bug in the program rather than an + // expected failure. + let key = key.expect("serialize_value called before serialize_key"); + self.map.insert(key, to_value(value)?); + Ok(()) + } + + fn end(self) -> Result { + Ok(Value::Object(self.map)) + } +} + +impl ser::SerializeStruct for SerializeMap { + type Ok = Value; + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeMap::serialize_entry(self, key, value) + } + + fn end(self) -> Result { + ser::SerializeMap::end(self) + } +} + +impl ser::SerializeStructVariant for SerializeStructVariant { + type Ok = Value; + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + ser::Serialize, + { + self.map.insert(key.to_owned(), to_value(&value)?); + Ok(()) + } + + fn end(self) -> Result { + let mut object = map_with_capacity(Some(1)); + + object.insert(self.name.to_owned(), Value::Object(self.map)); + + Ok(Value::Object(object)) + } +} + +impl<'de> de::Deserializer<'de> for Value { + type Error = Error; + + #[inline] + fn deserialize_any(self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + match self { + Value::Null => visitor.visit_unit(), + Value::Bool(v) => visitor.visit_bool(v), + Value::I64(v) => visitor.visit_i64(v), + Value::U64(v) => visitor.visit_u64(v), + Value::F64(v) => visitor.visit_f64(v), + Value::String(v) => visitor.visit_string(v), + Value::Array(v) => visitor.visit_seq(SeqDeserializer { + iter: v.into_iter(), + }), + Value::Object(v) => visitor.visit_map(MapDeserializer { + iter: v.into_iter(), + value: None, + }), + } + } + + #[inline] + fn deserialize_option(self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + match self { + Value::Null => visitor.visit_none(), + _ => visitor.visit_some(self), + } + } + + #[inline] + fn deserialize_enum( + self, + _name: &str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: de::Visitor<'de>, + { + let (variant, value) = match self { + Value::Object(value) => { + let mut iter = value.into_iter(); + let (variant, value) = match iter.next() { + Some(v) => v, + None => { + return Err(de::Error::invalid_type( + de::Unexpected::Map, + &"map with a single key", + )); + } + }; + // enums are encoded in json as maps with a single key:value pair + if iter.next().is_some() { + return Err(de::Error::invalid_type( + de::Unexpected::Map, + &"map with a single key", + )); + } + (variant, Some(value)) + } + Value::String(variant) => (variant, None), + val => { + return Err(de::Error::invalid_type( + val.as_unexpected(), + &"string or map", + )) + } + }; + + visitor.visit_enum(EnumDeserializer { variant, value }) + } + + #[inline] + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: de::Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string + bytes byte_buf unit unit_struct seq tuple + tuple_struct map struct identifier ignored_any + } +} + +struct EnumDeserializer { + variant: String, + value: Option, +} + +impl<'de> de::EnumAccess<'de> for EnumDeserializer { + type Error = Error; + + type Variant = VariantDeserializer; + + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: de::DeserializeSeed<'de>, + { + let variant = de::IntoDeserializer::into_deserializer(self.variant); + let visitor = VariantDeserializer { val: self.value }; + seed.deserialize(variant).map(|v| (v, visitor)) + } +} + +struct VariantDeserializer { + val: Option, +} + +impl<'de, 'a> de::VariantAccess<'de> for VariantDeserializer { + type Error = Error; + + fn unit_variant(self) -> Result<()> { + match self.val { + Some(val) => de::Deserialize::deserialize(val), + None => Ok(()), + } + } + + fn newtype_variant_seed(self, seed: T) -> Result + where + T: de::DeserializeSeed<'de>, + { + match self.val { + Some(value) => seed.deserialize(value), + None => Err(serde::de::Error::invalid_type( + de::Unexpected::UnitVariant, + &"newtype variant", + )), + } + } + + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + let val = self.val.expect("val is missing"); + if let Value::Array(fields) = val { + visitor.visit_seq(SeqDeserializer { + iter: fields.into_iter(), + }) + } else { + Err(de::Error::invalid_type(val.as_unexpected(), &visitor)) + } + } + + fn struct_variant(self, _fields: &'static [&'static str], visitor: V) -> Result + where + V: de::Visitor<'de>, + { + match self.val { + Some(Value::Object(fields)) => visitor.visit_map(MapDeserializer { + iter: fields.into_iter(), + value: None, + }), + Some(other) => Err(de::Error::invalid_type( + other.as_unexpected(), + &"struct variant", + )), + None => Err(de::Error::invalid_type( + de::Unexpected::UnitVariant, + &"struct variant", + )), + } + } +} + +struct SeqDeserializer { + iter: vec::IntoIter, +} + +impl<'de> de::SeqAccess<'de> for SeqDeserializer { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: de::DeserializeSeed<'de>, + { + match self.iter.next() { + Some(value) => Ok(Some(seed.deserialize(value)?)), + None => Ok(None), + } + } + + fn size_hint(&self) -> Option { + match self.iter.size_hint() { + (lower, Some(upper)) if lower == upper => Some(upper), + _ => None, + } + } +} + +struct MapDeserializer { + iter: MapIntoIter, + value: Option, +} + +impl<'de, 'a> de::MapAccess<'de> for MapDeserializer { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: de::DeserializeSeed<'de>, + { + match self.iter.next() { + Some((key, value)) => { + self.value = Some(value); + Ok(Some(seed.deserialize(Value::String(key))?)) + } + None => Ok(None), + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + let value = self.value.take().expect("value is missing"); + seed.deserialize(value) + } + + fn size_hint(&self) -> Option { + match self.iter.size_hint() { + (lower, Some(upper)) if lower == upper => Some(upper), + _ => None, + } + } +} + +pub fn to_value(value: &T) -> Result +where + T: ser::Serialize, +{ + value.serialize(Serializer) +} + +/// Shortcut function to decode a Hjson `Value` into a `T` +pub fn from_value(value: Value) -> Result +where + T: de::DeserializeOwned, +{ + de::Deserialize::deserialize(value) +} + +/// A trait for converting values to Hjson +pub trait ToJson { + /// Converts the value of `self` to an instance of Hjson + fn to_json(&self) -> Value; +} + +impl ToJson for T +where + T: ser::Serialize, +{ + fn to_json(&self) -> Value { + to_value(&self).expect("failed to serialize") + } +} + +#[cfg(test)] +mod test { + use super::Value; + use crate::de::from_str; + + #[test] + fn number_deserialize() { + let v: Value = from_str("{\"a\":1}").unwrap(); + let vo = v.as_object().unwrap(); + assert_eq!(vo["a"].as_u64().unwrap(), 1); + + let v: Value = from_str("{\"a\":-1}").unwrap(); + let vo = v.as_object().unwrap(); + assert_eq!(vo["a"].as_i64().unwrap(), -1); + + let v: Value = from_str("{\"a\":1.1}").unwrap(); + let vo = v.as_object().unwrap(); + assert!(vo["a"].as_f64().unwrap() - 1.1 < std::f64::EPSILON); + + let v: Value = from_str("{\"a\":-1.1}").unwrap(); + let vo = v.as_object().unwrap(); + assert!(vo["a"].as_f64().unwrap() + 1.1 > -(std::f64::EPSILON)); + + let v: Value = from_str("{\"a\":1e6}").unwrap(); + let vo = v.as_object().unwrap(); + assert!(vo["a"].as_f64().unwrap() - 1e6 < std::f64::EPSILON); + + let v: Value = from_str("{\"a\":-1e6}").unwrap(); + let vo = v.as_object().unwrap(); + assert!(vo["a"].as_f64().unwrap() + 1e6 > -(std::f64::EPSILON)); + } +} diff --git a/crates/nu-json/tests/main.rs b/crates/nu-json/tests/main.rs new file mode 100644 index 0000000000..28c264dc95 --- /dev/null +++ b/crates/nu-json/tests/main.rs @@ -0,0 +1,212 @@ +// FIXME: re-enable tests +/* +use nu_json::Value; +use regex::Regex; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +fn txt(text: &str) -> String { + let out = String::from_utf8_lossy(text.as_bytes()); + + #[cfg(windows)] + { + out.replace("\r\n", "").replace("\n", "") + } + + #[cfg(not(windows))] + { + out.to_string() + } +} + +fn hjson_expectations() -> PathBuf { + let assets = nu_test_support::fs::assets().join("nu_json"); + + nu_path::canonicalize(assets.clone()).unwrap_or_else(|e| { + panic!( + "Couldn't canonicalize hjson assets path {}: {:?}", + assets.display(), + e + ) + }) +} + +fn get_test_content(name: &str) -> io::Result { + let expectations = hjson_expectations(); + + let mut p = format!("{}/{}_test.hjson", expectations.display(), name); + + if !Path::new(&p).exists() { + p = format!("{}/{}_test.json", expectations.display(), name); + } + + fs::read_to_string(&p) +} + +fn get_result_content(name: &str) -> io::Result<(String, String)> { + let expectations = hjson_expectations(); + + let p1 = format!("{}/{}_result.json", expectations.display(), name); + let p2 = format!("{}/{}_result.hjson", expectations.display(), name); + + Ok((fs::read_to_string(&p1)?, fs::read_to_string(&p2)?)) +} + +macro_rules! run_test { + // {{ is a workaround for rust stable + ($v: ident, $list: expr, $fix: expr) => {{ + let name = stringify!($v); + $list.push(format!("{}_test", name)); + println!("- running {}", name); + let should_fail = name.starts_with("fail"); + let test_content = get_test_content(name).unwrap(); + let data: nu_json::Result = nu_json::from_str(&test_content); + assert!(should_fail == data.is_err()); + + if !should_fail { + let udata = data.unwrap(); + let (rjson, rhjson) = get_result_content(name).unwrap(); + let rjson = txt(&rjson); + let rhjson = txt(&rhjson); + let actual_hjson = nu_json::to_string(&udata).unwrap(); + let actual_hjson = txt(&actual_hjson); + let actual_json = $fix(serde_json::to_string_pretty(&udata).unwrap()); + let actual_json = txt(&actual_json); + if rhjson != actual_hjson { + println!( + "{:?}\n---hjson expected\n{}\n---hjson actual\n{}\n---\n", + name, rhjson, actual_hjson + ); + } + if rjson != actual_json { + println!( + "{:?}\n---json expected\n{}\n---json actual\n{}\n---\n", + name, rjson, actual_json + ); + } + assert!(rhjson == actual_hjson && rjson == actual_json); + } + }}; +} + +// add fixes where rust's json differs from javascript + +fn std_fix(json: String) -> String { + // serde_json serializes integers with a superfluous .0 suffix + let re = Regex::new(r"(?m)(?P\d)\.0(?P,?)$").unwrap(); + re.replace_all(&json, "$d$s").to_string() +} + +fn fix_kan(json: String) -> String { + std_fix(json).replace(" -0,", " 0,") +} + +fn fix_pass1(json: String) -> String { + std_fix(json) + .replace("1.23456789e34", "1.23456789e+34") + .replace("2.3456789012e76", "2.3456789012e+76") +} + +#[test] +fn test_hjson() { + let mut done: Vec = Vec::new(); + + println!(); + run_test!(charset, done, std_fix); + run_test!(comments, done, std_fix); + run_test!(empty, done, std_fix); + run_test!(failCharset1, done, std_fix); + run_test!(failJSON02, done, std_fix); + run_test!(failJSON05, done, std_fix); + run_test!(failJSON06, done, std_fix); + run_test!(failJSON07, done, std_fix); + run_test!(failJSON08, done, std_fix); + run_test!(failJSON10, done, std_fix); + run_test!(failJSON11, done, std_fix); + run_test!(failJSON12, done, std_fix); + run_test!(failJSON13, done, std_fix); + run_test!(failJSON14, done, std_fix); + run_test!(failJSON15, done, std_fix); + run_test!(failJSON16, done, std_fix); + run_test!(failJSON17, done, std_fix); + run_test!(failJSON19, done, std_fix); + run_test!(failJSON20, done, std_fix); + run_test!(failJSON21, done, std_fix); + run_test!(failJSON22, done, std_fix); + run_test!(failJSON23, done, std_fix); + run_test!(failJSON24, done, std_fix); + run_test!(failJSON26, done, std_fix); + run_test!(failJSON28, done, std_fix); + run_test!(failJSON29, done, std_fix); + run_test!(failJSON30, done, std_fix); + run_test!(failJSON31, done, std_fix); + run_test!(failJSON32, done, std_fix); + run_test!(failJSON33, done, std_fix); + run_test!(failJSON34, done, std_fix); + run_test!(failKey1, done, std_fix); + run_test!(failKey2, done, std_fix); + run_test!(failKey3, done, std_fix); + run_test!(failKey4, done, std_fix); + run_test!(failMLStr1, done, std_fix); + run_test!(failObj1, done, std_fix); + run_test!(failObj2, done, std_fix); + run_test!(failObj3, done, std_fix); + run_test!(failStr1a, done, std_fix); + run_test!(failStr1b, done, std_fix); + run_test!(failStr1c, done, std_fix); + run_test!(failStr1d, done, std_fix); + run_test!(failStr2a, done, std_fix); + run_test!(failStr2b, done, std_fix); + run_test!(failStr2c, done, std_fix); + run_test!(failStr2d, done, std_fix); + run_test!(failStr3a, done, std_fix); + run_test!(failStr3b, done, std_fix); + run_test!(failStr3c, done, std_fix); + run_test!(failStr3d, done, std_fix); + run_test!(failStr4a, done, std_fix); + run_test!(failStr4b, done, std_fix); + run_test!(failStr4c, done, std_fix); + run_test!(failStr4d, done, std_fix); + run_test!(failStr5a, done, std_fix); + run_test!(failStr5b, done, std_fix); + run_test!(failStr5c, done, std_fix); + run_test!(failStr5d, done, std_fix); + run_test!(failStr6a, done, std_fix); + run_test!(failStr6b, done, std_fix); + run_test!(failStr6c, done, std_fix); + run_test!(failStr6d, done, std_fix); + run_test!(kan, done, fix_kan); + run_test!(keys, done, std_fix); + run_test!(oa, done, std_fix); + run_test!(pass1, done, fix_pass1); + run_test!(pass2, done, std_fix); + run_test!(pass3, done, std_fix); + run_test!(pass4, done, std_fix); + run_test!(passSingle, done, std_fix); + run_test!(root, done, std_fix); + run_test!(stringify1, done, std_fix); + run_test!(strings, done, std_fix); + run_test!(trail, done, std_fix); + + // check if we include all assets + let paths = fs::read_dir(hjson_expectations()).unwrap(); + + let all = paths + .map(|item| String::from(item.unwrap().path().file_stem().unwrap().to_str().unwrap())) + .filter(|x| x.contains("_test")); + + let missing = all + .into_iter() + .filter(|x| done.iter().find(|y| &x == y) == None) + .collect::>(); + + if !missing.is_empty() { + for item in missing { + println!("missing: {}", item); + } + panic!(); + } +} + +*/ diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index f0ca45a1fb..96dd9444a5 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -4,5 +4,6 @@ version = "0.1.0" edition = "2018" [dependencies] -codespan-reporting = "0.11.1" -nu-protocol = { path = "../nu-protocol"} \ No newline at end of file +miette = "3.0.0" +thiserror = "1.0.29" +nu-protocol = { path = "../nu-protocol"} diff --git a/crates/nu-parser/README.md b/crates/nu-parser/README.md new file mode 100644 index 0000000000..58681f88ba --- /dev/null +++ b/crates/nu-parser/README.md @@ -0,0 +1,99 @@ +# nu-parser, the Nushell parser + +Nushell's parser is a type-directed parser, meaning that the parser will use type information available during parse time to configure the parser. This allows it to handle a broader range of techniques to handle the arguments of a command. + +Nushell's base language is whitespace-separated tokens with the command (Nushell's term for a function) name in the head position: + +``` +head1 arg1 arg2 | head2 +``` + +## Lexing + +The first job of the parser is to a lexical analysis to find where the tokens start and end in the input. This turns the above into: + +``` +, , , , +``` + +At this point, the parser has little to no understanding of the shape of the command or how to parse its arguments. + +## Lite parsing + +As nushell is a language of pipelines, pipes form a key role in both separating commands from each other as well as denoting the flow of information between commands. The lite parse phase, as the name suggests, helps to group the lexed tokens into units. + +The above tokens are converted the following during the lite parse phase: + +``` +Pipeline: + Command #1: + , , + Command #2: + +``` + +## Parsing + +The real magic begins to happen when the parse moves on to the parsing stage. At this point, it traverses the lite parse tree and for each command makes a decision: + +* If the command looks like an internal/external command literal: eg) `foo` or `/usr/bin/ls`, it parses it as an internal or external command +* Otherwise, it parses the command as part of a mathematical expression + +### Types/shapes + +Each command has a shape assigned to each of the arguments in reads in. These shapes help define how the parser will handle the parse. + +For example, if the command is written as: + +```sql +where $x > 10 +``` + +When the parsing happens, the parser will look up the `where` command and find its Signature. The Signature states what flags are allowed and what positional arguments are allowed (both required and optional). Each argument comes with it a Shape that defines how to parse values to get that position. + +In the above example, if the Signature of `where` said that it took three String values, the result would be: + +``` +CallInfo: + Name: `where` + Args: + Expression($x), a String + Expression(>), a String + Expression(10), a String +``` + +Or, the Signature could state that it takes in three positional arguments: a Variable, an Operator, and a Number, which would give: + +``` +CallInfo: + Name: `where` + Args: + Expression($x), a Variable + Expression(>), an Operator + Expression(10), a Number +``` + +Note that in this case, each would be checked at compile time to confirm that the expression has the shape requested. For example, `"foo"` would fail to parse as a Number. + +Finally, some Shapes can consume more than one token. In the above, if the `where` command stated it took in a single required argument, and that the Shape of this argument was a MathExpression, then the parser would treat the remaining tokens as part of the math expression. + +``` +CallInfo: + Name: `where` + Args: + MathExpression: + Op: > + LHS: Expression($x) + RHS: Expression(10) +``` + +When the command runs, it will now be able to evaluate the whole math expression as a single step rather than doing any additional parsing to understand the relationship between the parameters. + +## Making space + +As some Shapes can consume multiple tokens, it's important that the parser allow for multiple Shapes to coexist as peacefully as possible. + +The simplest way it does this is to ensure there is at least one token for each required parameter. If the Signature of the command says that it takes a MathExpression and a Number as two required arguments, then the parser will stop the math parser one token short. This allows the second Shape to consume the final token. + +Another way that the parser makes space is to look for Keyword shapes in the Signature. A Keyword is a word that's special to this command. For example in the `if` command, `else` is a keyword. When it is found in the arguments, the parser will use it as a signpost for where to make space for each Shape. The tokens leading up to the `else` will then feed into the parts of the Signature before the `else`, and the tokens following are consumed by the `else` and the Shapes that follow. + diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index bcd388c279..58123aac0d 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,31 +1,162 @@ +use miette::Diagnostic; use nu_protocol::{Span, Type}; +use thiserror::Error; -#[derive(Debug)] +#[derive(Clone, Debug, Error, Diagnostic)] pub enum ParseError { - ExtraTokens(Span), - ExtraPositional(Span), - UnexpectedEof(String, Span), - Unclosed(String, Span), - UnknownStatement(Span), - Expected(String, Span), - Mismatch(String, String, Span), // expected, found, span - UnsupportedOperation(Span, Span, Type, Span, Type), - ExpectedKeyword(String, Span), - MultipleRestParams(Span), - VariableNotFound(Span), - UnknownCommand(Span), - NonUtf8(Span), - UnknownFlag(Span), - UnknownType(Span), - MissingFlagParam(Span), - ShortFlagBatchCantTakeArg(Span), - MissingPositional(String, Span), - KeywordMissingArgument(String, Span), - MissingType(Span), - TypeMismatch(Type, Type, Span), // expected, found, span - MissingRequiredFlag(String, Span), - IncompleteMathExpression(Span), - UnknownState(String, Span), - IncompleteParser(Span), - RestNeedsName(Span), + /// The parser encountered unexpected tokens, when the code should have + /// finished. You should remove these or finish adding what you intended + /// to add. + #[error("Extra tokens in code.")] + #[diagnostic( + code(nu::parser::extra_tokens), + url(docsrs), + help("Try removing them.") + )] + ExtraTokens(#[label = "extra tokens"] Span), + + #[error("Extra positional argument.")] + #[diagnostic(code(nu::parser::extra_positional), url(docsrs))] + ExtraPositional(#[label = "extra positional argument"] Span), + + #[error("Unexpected end of code.")] + #[diagnostic(code(nu::parser::unexpected_eof), url(docsrs))] + UnexpectedEof(String, #[label("expected closing {0}")] Span), + + #[error("Unclosed delimiter.")] + #[diagnostic(code(nu::parser::unclosed_delimiter), url(docsrs))] + Unclosed(String, #[label("unclosed {0}")] Span), + + #[error("Unknown statement.")] + #[diagnostic(code(nu::parser::unknown_statement), url(docsrs))] + UnknownStatement(#[label("unknown statement")] Span), + + #[error("Parse mismatch during operation.")] + #[diagnostic(code(nu::parser::parse_mismatch), url(docsrs))] + Expected(String, #[label("expected {0}")] Span), + + #[error("Type mismatch during operation.")] + #[diagnostic(code(nu::parser::type_mismatch), url(docsrs))] + Mismatch(String, String, #[label("expected {0}, found {1}")] Span), // expected, found, span + + #[error("Types mismatched for operation.")] + #[diagnostic( + code(nu::parser::unsupported_operation), + url(docsrs), + help("Change {2} or {4} to be the right types and try again.") + )] + UnsupportedOperation( + #[label = "doesn't support these values."] Span, + #[label("{2}")] Span, + Type, + #[label("{4}")] Span, + Type, + ), + + #[error("Expected keyword.")] + #[diagnostic(code(nu::parser::expected_keyword), url(docsrs))] + ExpectedKeyword(String, #[label("expected {0}")] Span), + + #[error("Multiple rest params.")] + #[diagnostic(code(nu::parser::multiple_rest_params), url(docsrs))] + MultipleRestParams(#[label = "multiple rest params"] Span), + + #[error("Variable not found.")] + #[diagnostic(code(nu::parser::variable_not_found), url(docsrs))] + VariableNotFound(#[label = "variable not found"] Span), + + #[error("Module not found.")] + #[diagnostic(code(nu::parser::module_not_found), url(docsrs))] + ModuleNotFound(#[label = "module not found"] Span), + + #[error("Unknown command.")] + #[diagnostic( + code(nu::parser::unknown_command), + url(docsrs), + // TODO: actual suggestions + // help("Did you mean `foo`?") + )] + UnknownCommand(#[label = "unknown command"] Span), + + #[error("Non-UTF8 code.")] + #[diagnostic(code(nu::parser::non_utf8), url(docsrs))] + NonUtf8(#[label = "non-UTF8 code"] Span), + + #[error("The `{0}` command doesn't have flag `{1}`.")] + #[diagnostic(code(nu::parser::unknown_flag), url(docsrs))] + UnknownFlag(String, String, #[label = "unknown flag"] Span), + + #[error("Unknown type.")] + #[diagnostic(code(nu::parser::unknown_type), url(docsrs))] + UnknownType(#[label = "unknown type"] Span), + + #[error("Missing flag param.")] + #[diagnostic(code(nu::parser::missing_flag_param), url(docsrs))] + MissingFlagParam(#[label = "flag missing param"] Span), + + #[error("Batches of short flags can't take arguments.")] + #[diagnostic(code(nu::parser::short_flag_arg_cant_take_arg), url(docsrs))] + ShortFlagBatchCantTakeArg(#[label = "short flag batches can't take args"] Span), + + #[error("Missing required positional argument.")] + #[diagnostic(code(nu::parser::missing_positional), url(docsrs))] + MissingPositional(String, #[label("missing {0}")] Span), + + #[error("Missing argument to `{0}`.")] + #[diagnostic(code(nu::parser::keyword_missing_arg), url(docsrs))] + KeywordMissingArgument(String, #[label("missing value that follows {0}")] Span), + + #[error("Missing type.")] + #[diagnostic(code(nu::parser::missing_type), url(docsrs))] + MissingType(#[label = "expected type"] Span), + + #[error("Type mismatch.")] + #[diagnostic(code(nu::parser::type_mismatch), url(docsrs))] + TypeMismatch(Type, Type, #[label("expected {0:?}, found {1:?}")] Span), // expected, found, span + + #[error("Missing required flag.")] + #[diagnostic(code(nu::parser::missing_required_flag), url(docsrs))] + MissingRequiredFlag(String, #[label("missing required flag {0}")] Span), + + #[error("Incomplete math expression.")] + #[diagnostic(code(nu::parser::incomplete_math_expression), url(docsrs))] + IncompleteMathExpression(#[label = "incomplete math expression"] Span), + + #[error("Unknown state.")] + #[diagnostic(code(nu::parser::unknown_state), url(docsrs))] + UnknownState(String, #[label("{0}")] Span), + + #[error("Parser incomplete.")] + #[diagnostic(code(nu::parser::parser_incomplete), url(docsrs))] + IncompleteParser(#[label = "parser support missing for this expression"] Span), + + #[error("Rest parameter needs a name.")] + #[diagnostic(code(nu::parser::rest_needs_name), url(docsrs))] + RestNeedsName(#[label = "needs a parameter name"] Span), + + #[error("Extra columns.")] + #[diagnostic(code(nu::parser::extra_columns), url(docsrs))] + ExtraColumns( + usize, + #[label("expected {0} column{}", if *.0 == 1 { "" } else { "s" })] Span, + ), + + #[error("Missing columns.")] + #[diagnostic(code(nu::parser::missing_columns), url(docsrs))] + MissingColumns( + usize, + #[label("expected {0} column{}", if *.0 == 1 { "" } else { "s" })] Span, + ), + + #[error("{0}")] + #[diagnostic(code(nu::parser::assignment_mismatch), url(docsrs))] + AssignmentMismatch(String, String, #[label("{1}")] Span), + + #[error("Missing import pattern.")] + #[diagnostic(code(nu::parser::missing_import_pattern), url(docsrs))] + MissingImportPattern(#[label = "needs an import pattern"] Span), + + #[error("Module export not found.")] + #[diagnostic(code(nu::parser::export_not_found), url(docsrs))] + ExportNotFound(#[label = "could not find imports"] Span), } diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index 7439da6181..7b94d007c9 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -10,11 +10,13 @@ pub enum FlatShape { Range, InternalCall, External, + ExternalArg, Literal, Operator, Signature, String, Variable, + Custom(String), } pub fn flatten_block(working_set: &StateWorkingSet, block: &Block) -> Vec<(Span, FlatShape)> { @@ -39,6 +41,10 @@ pub fn flatten_expression( working_set: &StateWorkingSet, expr: &Expression, ) -> Vec<(Span, FlatShape)> { + if let Some(custom_completion) = &expr.custom_completion { + return vec![(expr.span, FlatShape::Custom(custom_completion.clone()))]; + } + match &expr.expr { Expr::BinaryOp(lhs, op, rhs) => { let mut output = vec![]; @@ -55,8 +61,14 @@ pub fn flatten_expression( } output } - Expr::ExternalCall(..) => { - vec![(expr.span, FlatShape::External)] + Expr::ExternalCall(name, args) => { + let mut output = vec![(*name, FlatShape::External)]; + + for arg in args { + output.push((*arg, FlatShape::ExternalArg)); + } + + output } Expr::Garbage => { vec![(expr.span, FlatShape::Garbage)] @@ -67,10 +79,10 @@ pub fn flatten_expression( Expr::Float(_) => { vec![(expr.span, FlatShape::Float)] } - Expr::FullCellPath(column_path) => { + Expr::FullCellPath(cell_path) => { let mut output = vec![]; - output.extend(flatten_expression(working_set, &column_path.head)); - for path_element in &column_path.tail { + output.extend(flatten_expression(working_set, &cell_path.head)); + for path_element in &cell_path.tail { match path_element { PathMember::String { span, .. } => output.push((*span, FlatShape::String)), PathMember::Int { span, .. } => output.push((*span, FlatShape::Int)), @@ -78,15 +90,19 @@ pub fn flatten_expression( } output } - Expr::Range(from, to, op) => { + Expr::Range(from, next, to, op) => { let mut output = vec![]; if let Some(f) = from { output.extend(flatten_expression(working_set, f)); } + if let Some(s) = next { + output.extend(vec![(op.next_op_span, FlatShape::Operator)]); + output.extend(flatten_expression(working_set, s)); + } + output.extend(vec![(op.span, FlatShape::Operator)]); if let Some(t) = to { output.extend(flatten_expression(working_set, t)); } - output.extend(vec![(op.span, FlatShape::Operator)]); output } Expr::Bool(_) => { @@ -114,6 +130,7 @@ pub fn flatten_expression( Expr::String(_) => { vec![(expr.span, FlatShape::String)] } + Expr::RowCondition(_, expr) => flatten_expression(working_set, expr), Expr::Subexpression(block_id) => { flatten_block(working_set, working_set.get_block(*block_id)) } diff --git a/crates/nu-parser/src/lex.rs b/crates/nu-parser/src/lex.rs index 867fdb4839..b1b50f99be 100644 --- a/crates/nu-parser/src/lex.rs +++ b/crates/nu-parser/src/lex.rs @@ -168,7 +168,7 @@ pub fn lex_item( (delim as char).to_string(), Span { start: span.end, - end: span.end + 1, + end: span.end, }, ); @@ -181,7 +181,13 @@ pub fn lex_item( // correct information from the non-lite parse. return ( span, - Some(ParseError::UnexpectedEof((delim as char).to_string(), span)), + Some(ParseError::UnexpectedEof( + (delim as char).to_string(), + Span { + start: span.end, + end: span.end, + }, + )), ); } diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index cfd9535f70..0fd2f3ec8c 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -2,6 +2,7 @@ mod errors; mod flatten; mod lex; mod lite_parse; +mod parse_keywords; mod parser; mod type_check; @@ -9,4 +10,7 @@ pub use errors::ParseError; pub use flatten::{flatten_block, FlatShape}; pub use lex::{lex, Token, TokenContents}; pub use lite_parse::{lite_parse, LiteBlock}; +pub use parse_keywords::{ + parse_alias, parse_def, parse_def_predecl, parse_let, parse_module, parse_use, +}; pub use parser::{parse, Import, VarDecl}; diff --git a/crates/nu-parser/src/parse_keywords.rs b/crates/nu-parser/src/parse_keywords.rs new file mode 100644 index 0000000000..e516da309b --- /dev/null +++ b/crates/nu-parser/src/parse_keywords.rs @@ -0,0 +1,541 @@ +use nu_protocol::{ + ast::{Block, Call, Expr, Expression, ImportPatternMember, Pipeline, Statement}, + engine::StateWorkingSet, + span, DeclId, Span, SyntaxShape, Type, +}; + +use crate::{ + lex, lite_parse, + parser::{ + check_name, garbage, garbage_statement, parse_block_expression, parse_import_pattern, + parse_internal_call, parse_signature, parse_string, + }, + ParseError, +}; + +pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"def" && spans.len() >= 4 { + let (name_expr, ..) = parse_string(working_set, spans[1]); + let name = name_expr.as_string(); + + working_set.enter_scope(); + // FIXME: because parse_signature will update the scope with the variables it sees + // we end up parsing the signature twice per def. The first time is during the predecl + // so that we can see the types that are part of the signature, which we need for parsing. + // The second time is when we actually parse the body itworking_set. + // We can't reuse the first time because the variables that are created during parse_signature + // are lost when we exit the scope below. + let (sig, ..) = parse_signature(working_set, spans[2]); + let signature = sig.as_signature(); + working_set.exit_scope(); + + if let (Some(name), Some(mut signature)) = (name, signature) { + signature.name = name; + let decl = signature.predeclare(); + + working_set.add_decl(decl); + } + } +} + +pub fn parse_def( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let mut error = None; + let name = working_set.get_span_contents(spans[0]); + + if name == b"def" { + let def_decl_id = working_set + .find_decl(b"def") + .expect("internal error: missing def command"); + + let mut call = Box::new(Call { + head: spans[0], + decl_id: def_decl_id, + positional: vec![], + named: vec![], + }); + + let call = if let Some(name_span) = spans.get(1) { + let (name_expr, err) = parse_string(working_set, *name_span); + error = error.or(err); + + let name = name_expr.as_string(); + call.positional.push(name_expr); + + if let Some(sig_span) = spans.get(2) { + working_set.enter_scope(); + let (sig, err) = parse_signature(working_set, *sig_span); + error = error.or(err); + + let signature = sig.as_signature(); + + call.positional.push(sig); + + if let Some(block_span) = spans.get(3) { + let (block, err) = parse_block_expression( + working_set, + &SyntaxShape::Block(Some(vec![])), + *block_span, + ); + error = error.or(err); + + let block_id = block.as_block(); + + call.positional.push(block); + + if let (Some(name), Some(mut signature), Some(block_id)) = + (name, signature, block_id) + { + let decl_id = working_set + .find_decl(name.as_bytes()) + .expect("internal error: predeclaration failed to add definition"); + + let declaration = working_set.get_decl_mut(decl_id); + + signature.name = name; + + *declaration = signature.into_block_command(block_id); + } + } else { + let err_span = Span { + start: sig_span.end, + end: sig_span.end, + }; + + error = error + .or_else(|| Some(ParseError::MissingPositional("block".into(), err_span))); + } + working_set.exit_scope(); + + call + } else { + let err_span = Span { + start: name_span.end, + end: name_span.end, + }; + + error = error + .or_else(|| Some(ParseError::MissingPositional("parameters".into(), err_span))); + + call + } + } else { + let err_span = Span { + start: spans[0].end, + end: spans[0].end, + }; + + error = error.or_else(|| { + Some(ParseError::MissingPositional( + "definition name".into(), + err_span, + )) + }); + + call + }; + + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, + custom_completion: None, + }])), + error, + ) + } else { + ( + garbage_statement(spans), + Some(ParseError::UnknownState( + "Expected structure: def [] {}".into(), + span(spans), + )), + ) + } +} + +pub fn parse_alias( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"alias" { + if let Some((span, err)) = check_name(working_set, spans) { + return ( + Statement::Pipeline(Pipeline::from_vec(vec![garbage(*span)])), + Some(err), + ); + } + + if let Some(decl_id) = working_set.find_decl(b"alias") { + let (call, call_span, _) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + if spans.len() >= 4 { + let alias_name = working_set.get_span_contents(spans[1]); + + let alias_name = if alias_name.starts_with(b"\"") + && alias_name.ends_with(b"\"") + && alias_name.len() > 1 + { + alias_name[1..(alias_name.len() - 1)].to_vec() + } else { + alias_name.to_vec() + }; + let _equals = working_set.get_span_contents(spans[2]); + + let replacement = spans[3..].to_vec(); + + //println!("{:?} {:?}", alias_name, replacement); + + working_set.add_alias(alias_name, replacement); + } + + return ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + custom_completion: None, + }])), + None, + ); + } + } + + ( + garbage_statement(spans), + Some(ParseError::UnknownState( + "internal error: alias statement unparseable".into(), + span(spans), + )), + ) +} + +pub fn parse_module( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + // TODO: Currently, module is closing over its parent scope (i.e., defs in the parent scope are + // visible and usable in this module's scope). We might want to disable that. How? + + let mut error = None; + let bytes = working_set.get_span_contents(spans[0]); + + // parse_def() equivalent + if bytes == b"module" && spans.len() >= 3 { + let (module_name_expr, err) = parse_string(working_set, spans[1]); + error = error.or(err); + + let module_name = module_name_expr + .as_string() + .expect("internal error: module name is not a string"); + + // parse_block_expression() equivalent + let block_span = spans[2]; + let block_bytes = working_set.get_span_contents(block_span); + let mut start = block_span.start; + let mut end = block_span.end; + + if block_bytes.starts_with(b"{") { + start += 1; + } else { + return ( + garbage_statement(spans), + Some(ParseError::Expected("block".into(), block_span)), + ); + } + + if block_bytes.ends_with(b"}") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "}".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } + + let block_span = Span { start, end }; + + let source = working_set.get_span_contents(block_span); + + let (output, err) = lex(source, start, &[], &[]); + error = error.or(err); + + working_set.enter_scope(); + + // Do we need block parameters? + + let (output, err) = lite_parse(&output); + error = error.or(err); + + // We probably don't need $it + + // we're doing parse_block() equivalent + // let (mut output, err) = parse_block(working_set, &output, false); + + for pipeline in &output.block { + if pipeline.commands.len() == 1 { + parse_def_predecl(working_set, &pipeline.commands[0].parts); + } + } + + let mut exports: Vec<(Vec, DeclId)> = vec![]; + + let block: Block = output + .block + .iter() + .map(|pipeline| { + if pipeline.commands.len() == 1 { + // this one here is doing parse_statement() equivalent + // let (stmt, err) = parse_statement(working_set, &pipeline.commands[0].parts); + let name = working_set.get_span_contents(pipeline.commands[0].parts[0]); + + let (stmt, err) = match name { + // TODO: Here we can add other stuff that's alowed for modules + b"def" => { + let (stmt, err) = parse_def(working_set, &pipeline.commands[0].parts); + + if err.is_none() { + let decl_name = + working_set.get_span_contents(pipeline.commands[0].parts[1]); + + let decl_id = working_set + .find_decl(decl_name) + .expect("internal error: failed to find added declaration"); + + // TODO: Later, we want to put this behind 'export' + exports.push((decl_name.into(), decl_id)); + } + + (stmt, err) + } + _ => ( + garbage_statement(&pipeline.commands[0].parts), + Some(ParseError::Expected( + "def".into(), + pipeline.commands[0].parts[0], + )), + ), + }; + + if error.is_none() { + error = err; + } + + stmt + } else { + error = Some(ParseError::Expected("not a pipeline".into(), block_span)); + garbage_statement(spans) + } + }) + .into(); + + let block = block.with_exports(exports); + + working_set.exit_scope(); + + let block_id = working_set.add_module(&module_name, block); + + let block_expr = Expression { + expr: Expr::Block(block_id), + span: block_span, + ty: Type::Block, + custom_completion: None, + }; + + let module_decl_id = working_set + .find_decl(b"module") + .expect("internal error: missing module command"); + + let call = Box::new(Call { + head: spans[0], + decl_id: module_decl_id, + positional: vec![module_name_expr, block_expr], + named: vec![], + }); + + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, + custom_completion: None, + }])), + error, + ) + } else { + ( + garbage_statement(spans), + Some(ParseError::UnknownState( + "Expected structure: module {}".into(), + span(spans), + )), + ) + } +} + +pub fn parse_use( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let mut error = None; + let bytes = working_set.get_span_contents(spans[0]); + + // TODO: Currently, this directly imports the module's definitions into the current scope. + // Later, we want to put them behind the module's name and add selective importing + if bytes == b"use" && spans.len() >= 2 { + let (module_name_expr, err) = parse_string(working_set, spans[1]); + error = error.or(err); + + let (import_pattern, err) = parse_import_pattern(working_set, spans[1]); + error = error.or(err); + + let exports = if let Some(block_id) = working_set.find_module(&import_pattern.head) { + // TODO: Since we don't use the Block at all, we might just as well create a separate + // Module that holds only the exports, without having Blocks in the way. + working_set.get_block(block_id).exports.clone() + } else { + return ( + garbage_statement(spans), + Some(ParseError::ModuleNotFound(spans[1])), + ); + }; + + let exports = if import_pattern.members.is_empty() { + exports + .into_iter() + .map(|(name, id)| { + let mut new_name = import_pattern.head.to_vec(); + new_name.push(b'.'); + new_name.extend(&name); + (new_name, id) + }) + .collect() + } else { + match &import_pattern.members[0] { + ImportPatternMember::Glob { .. } => exports, + ImportPatternMember::Name { name, span } => { + let new_exports: Vec<(Vec, usize)> = + exports.into_iter().filter(|x| &x.0 == name).collect(); + + if new_exports.is_empty() { + error = error.or(Some(ParseError::ExportNotFound(*span))) + } + + new_exports + } + ImportPatternMember::List { names } => { + let mut output = vec![]; + + for (name, span) in names { + let mut new_exports: Vec<(Vec, usize)> = exports + .iter() + .filter_map(|x| if &x.0 == name { Some(x.clone()) } else { None }) + .collect(); + + if new_exports.is_empty() { + error = error.or(Some(ParseError::ExportNotFound(*span))) + } else { + output.append(&mut new_exports) + } + } + + output + } + } + }; + + // Extend the current scope with the module's exports + working_set.activate_overlay(exports); + + // Create the Use command call + let use_decl_id = working_set + .find_decl(b"use") + .expect("internal error: missing use command"); + + let call = Box::new(Call { + head: spans[0], + decl_id: use_decl_id, + positional: vec![module_name_expr], + named: vec![], + }); + + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, + custom_completion: None, + }])), + error, + ) + } else { + ( + garbage_statement(spans), + Some(ParseError::UnknownState( + "Expected structure: use ".into(), + span(spans), + )), + ) + } +} + +pub fn parse_let( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"let" { + if let Some((span, err)) = check_name(working_set, spans) { + return ( + Statement::Pipeline(Pipeline::from_vec(vec![garbage(*span)])), + Some(err), + ); + } + + if let Some(decl_id) = working_set.find_decl(b"let") { + let (call, call_span, err) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + // Update the variable to the known type if we can. + if err.is_none() { + let var_id = call.positional[0] + .as_var() + .expect("internal error: expected variable"); + let rhs_type = call.positional[1].ty.clone(); + + working_set.set_variable_type(var_id, rhs_type); + } + + return ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + custom_completion: None, + }])), + err, + ); + } + } + ( + garbage_statement(spans), + Some(ParseError::UnknownState( + "internal error: let statement unparseable".into(), + span(spans), + )), + ) +} diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 38b5123763..d8773e4bfa 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -8,13 +8,17 @@ use crate::{ use nu_protocol::{ ast::{ - Block, Call, Expr, Expression, FullCellPath, Operator, PathMember, Pipeline, - RangeInclusion, RangeOperator, Statement, + Block, Call, Expr, Expression, FullCellPath, ImportPattern, ImportPatternMember, Operator, + PathMember, Pipeline, RangeInclusion, RangeOperator, Statement, }, engine::StateWorkingSet, span, Flag, PositionalArg, ShellError, Signature, Span, SyntaxShape, Type, VarId, }; +use crate::parse_keywords::{ + parse_alias, parse_def, parse_def_predecl, parse_let, parse_module, parse_use, +}; + #[derive(Debug, Clone)] pub enum Import {} @@ -24,10 +28,14 @@ pub struct VarDecl { expression: Expression, } -fn garbage(span: Span) -> Expression { +pub fn garbage(span: Span) -> Expression { Expression::garbage(span) } +pub fn garbage_statement(spans: &[Span]) -> Statement { + Statement::Pipeline(Pipeline::from_vec(vec![garbage(span(spans))])) +} + fn is_identifier_byte(b: u8) -> bool { b != b'.' && b != b'[' && b != b'(' && b != b'{' } @@ -61,21 +69,57 @@ fn check_call(command: Span, sig: &Signature, call: &Call) -> Option } } -pub fn parse_external_call( +pub fn check_name<'a>( working_set: &mut StateWorkingSet, + spans: &'a [Span], +) -> Option<(&'a Span, ParseError)> { + if spans.len() == 1 { + None + } else if spans.len() < 4 { + if working_set.get_span_contents(spans[1]) == b"=" { + let name = String::from_utf8_lossy(working_set.get_span_contents(spans[0])); + Some(( + &spans[1], + ParseError::AssignmentMismatch( + format!("{} missing name", name), + "missing name".into(), + spans[1], + ), + )) + } else { + None + } + } else if working_set.get_span_contents(spans[2]) != b"=" { + let name = String::from_utf8_lossy(working_set.get_span_contents(spans[0])); + Some(( + &spans[2], + ParseError::AssignmentMismatch( + format!("{} missing sign", name), + "missing equal sign".into(), + spans[2], + ), + )) + } else { + None + } +} + +pub fn parse_external_call( + _working_set: &mut StateWorkingSet, spans: &[Span], ) -> (Expression, Option) { // TODO: add external parsing let mut args = vec![]; - let name = working_set.get_span_contents(spans[0]).to_vec(); + let name = spans[0]; for span in &spans[1..] { - args.push(working_set.get_span_contents(*span).to_vec()); + args.push(*span); } ( Expression { expr: Expr::ExternalCall(name, args), span: span(spans), ty: Type::Unknown, + custom_completion: None, }, None, ) @@ -122,9 +166,13 @@ fn parse_long_flag( } } else { ( - Some(long_name), + Some(long_name.clone()), None, - Some(ParseError::UnknownFlag(arg_span)), + Some(ParseError::UnknownFlag( + sig.name.clone(), + long_name.clone(), + arg_span, + )), ) } } else { @@ -176,17 +224,45 @@ fn parse_short_flags( if String::from_utf8_lossy(arg_contents).parse::().is_ok() { return (None, None); } else if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); + let contents = working_set.get_span_contents(*first); + error = error.or_else(|| { + Some(ParseError::UnknownFlag( + sig.name.clone(), + format!("-{}", String::from_utf8_lossy(contents).to_string()), + *first, + )) + }); } } else if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); + let contents = working_set.get_span_contents(*first); + error = error.or_else(|| { + Some(ParseError::UnknownFlag( + sig.name.clone(), + format!("-{}", String::from_utf8_lossy(contents).to_string()), + *first, + )) + }); } } else if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); + let contents = working_set.get_span_contents(*first); + error = error.or_else(|| { + Some(ParseError::UnknownFlag( + sig.name.clone(), + format!("-{}", String::from_utf8_lossy(contents).to_string()), + *first, + )) + }); } } else if !unmatched_short_flags.is_empty() { if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); + let contents = working_set.get_span_contents(*first); + error = error.or_else(|| { + Some(ParseError::UnknownFlag( + sig.name.clone(), + format!("-{}", String::from_utf8_lossy(contents).to_string()), + *first, + )) + }); } } @@ -323,6 +399,7 @@ fn parse_multispan_value( ), span: arg_span, ty: Type::Unknown, + custom_completion: None, }, error, ); @@ -337,6 +414,7 @@ fn parse_multispan_value( expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)), span: arg_span, ty, + custom_completion: None, }, error, ) @@ -516,12 +594,14 @@ pub fn parse_call( expr: Expr::Call(mut call), span, ty, + custom_completion: None, } => { call.head = orig_span; Expression { expr: Expr::Call(call), span, ty, + custom_completion: None, } } x => x, @@ -559,12 +639,14 @@ pub fn parse_call( expr: Expr::Call(mut call), span, ty, + custom_completion: None, } => { call.head = orig_span; Expression { expr: Expr::Call(call), span, ty, + custom_completion: None, } } x => x, @@ -582,6 +664,23 @@ pub fn parse_call( name = new_name; pos += 1; } + + // Before the internal parsing we check if there is no let or alias declarations + // that are missing their name, e.g.: let = 1 or alias = 2 + if spans.len() > 1 { + let test_equal = working_set.get_span_contents(spans[1]); + + if test_equal == [b'='] { + return ( + garbage(Span::new(0, 0)), + Some(ParseError::UnknownState( + "Incomplete statement".into(), + span(spans), + )), + ); + } + } + // parse internal command let (call, _, err) = parse_internal_call(working_set, span(&spans[0..pos]), &spans[pos..], decl_id); @@ -590,10 +689,19 @@ pub fn parse_call( expr: Expr::Call(call), span: span(spans), ty: Type::Unknown, // FIXME + custom_completion: None, }, err, ) } else { + // We might be parsing left-unbounded range ("..10") + let bytes = working_set.get_span_contents(spans[0]); + if let (Some(b'.'), Some(b'.')) = (bytes.get(0), bytes.get(1)) { + let (range_expr, range_err) = parse_range(working_set, spans[0]); + if range_err.is_none() { + return (range_expr, range_err); + } + } parse_external_call(working_set, spans) } } @@ -606,6 +714,7 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option) { expr: Expr::Int(v), span, ty: Type::Int, + custom_completion: None, }, None, ) @@ -626,6 +735,7 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option) { expr: Expr::Int(v), span, ty: Type::Int, + custom_completion: None, }, None, ) @@ -646,6 +756,7 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option) { expr: Expr::Int(v), span, ty: Type::Int, + custom_completion: None, }, None, ) @@ -665,6 +776,7 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option) { expr: Expr::Int(x), span, ty: Type::Int, + custom_completion: None, }, None, ) @@ -683,6 +795,7 @@ pub fn parse_float(token: &[u8], span: Span) -> (Expression, Option) expr: Expr::Float(x), span, ty: Type::Float, + custom_completion: None, }, None, ) @@ -711,8 +824,8 @@ pub fn parse_range( working_set: &mut StateWorkingSet, span: Span, ) -> (Expression, Option) { - // Range follows the following syntax: [][][] - // where is ".." + // Range follows the following syntax: [][][] + // where is ".." // and is ".." or "..<" // and one of the or bounds must be present (just '..' is not allowed since it // looks like parent directory) @@ -727,42 +840,28 @@ pub fn parse_range( // First, figure out what exact operators are used and determine their positions let dotdot_pos: Vec<_> = token.match_indices("..").map(|(pos, _)| pos).collect(); - let (step_op_pos, range_op_pos) = + let (next_op_pos, range_op_pos) = match dotdot_pos.len() { 1 => (None, dotdot_pos[0]), 2 => (Some(dotdot_pos[0]), dotdot_pos[1]), _ => return ( garbage(span), Some(ParseError::Expected( - "one range operator ('..' or '..<') and optionally one step operator ('..')" + "one range operator ('..' or '..<') and optionally one next operator ('..')" .into(), span, )), ), }; - let _step_op_span = step_op_pos.map(|pos| { - Span::new( - span.start + pos, - span.start + pos + "..".len(), // Only ".." is allowed for step operator - ) - }); - - let (range_op, range_op_str, range_op_span) = if let Some(pos) = token.find("..<") { + let (inclusion, range_op_str, range_op_span) = if let Some(pos) = token.find("..<") { if pos == range_op_pos { let op_str = "..<"; let op_span = Span::new( span.start + range_op_pos, span.start + range_op_pos + op_str.len(), ); - ( - RangeOperator { - inclusion: RangeInclusion::RightExclusive, - span: op_span, - }, - "..<", - op_span, - ) + (RangeInclusion::RightExclusive, "..<", op_span) } else { return ( garbage(span), @@ -778,21 +877,14 @@ pub fn parse_range( span.start + range_op_pos, span.start + range_op_pos + op_str.len(), ); - ( - RangeOperator { - inclusion: RangeInclusion::Inclusive, - span: op_span, - }, - "..", - op_span, - ) + (RangeInclusion::Inclusive, "..", op_span) }; - // Now, based on the operator positions, figure out where the bounds & step are located and + // Now, based on the operator positions, figure out where the bounds & next are located and // parse them - // TODO: Actually parse the step number + // TODO: Actually parse the next number let from = if token.starts_with("..") { - // token starts with either step operator, or range operator -- we don't care which one + // token starts with either next operator, or range operator -- we don't care which one None } else { let from_span = Span::new(span.start, span.start + dotdot_pos[0]); @@ -832,11 +924,35 @@ pub fn parse_range( ); } + let (next, next_op_span) = if let Some(pos) = next_op_pos { + let next_op_span = Span::new(span.start + pos, span.start + pos + "..".len()); + let next_span = Span::new(next_op_span.end, range_op_span.start); + + match parse_value(working_set, next_span, &SyntaxShape::Number) { + (expression, None) => (Some(Box::new(expression)), next_op_span), + _ => { + return ( + garbage(span), + Some(ParseError::Expected("number".into(), span)), + ) + } + } + } else { + (None, Span::unknown()) + }; + + let range_op = RangeOperator { + inclusion, + span: range_op_span, + next_op_span, + }; + ( Expression { - expr: Expr::Range(from, to, range_op), + expr: Expr::Range(from, next, to, range_op), span, ty: Type::Range, + custom_completion: None, }, None, ) @@ -853,7 +969,7 @@ pub(crate) fn parse_dollar_expr( } else if let (expr, None) = parse_range(working_set, span) { (expr, None) } else { - parse_full_column_path(working_set, span) + parse_full_cell_path(working_set, None, span) } } @@ -907,6 +1023,7 @@ pub fn parse_string_interpolation( expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), span, ty: Type::String, + custom_completion: None, }); } token_start = b; @@ -924,7 +1041,7 @@ pub fn parse_string_interpolation( end: b + 1, }; - let (expr, err) = parse_full_column_path(working_set, span); + let (expr, err) = parse_full_cell_path(working_set, None, span); error = error.or(err); output.push(expr); } @@ -949,6 +1066,7 @@ pub fn parse_string_interpolation( expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), span, ty: Type::String, + custom_completion: None, }); } } @@ -959,7 +1077,7 @@ pub fn parse_string_interpolation( end, }; - let (expr, err) = parse_full_column_path(working_set, span); + let (expr, err) = parse_full_cell_path(working_set, None, span); error = error.or(err); output.push(expr); } @@ -980,6 +1098,7 @@ pub fn parse_string_interpolation( })), span, ty: Type::String, + custom_completion: None, }, error, ) @@ -1003,6 +1122,7 @@ pub fn parse_variable_expr( expr: Expr::Bool(true), span, ty: Type::Bool, + custom_completion: None, }, None, ); @@ -1012,6 +1132,7 @@ pub fn parse_variable_expr( expr: Expr::Bool(false), span, ty: Type::Bool, + custom_completion: None, }, None, ); @@ -1026,43 +1147,35 @@ pub fn parse_variable_expr( expr: Expr::Var(id), span, ty: working_set.get_variable(id).clone(), + custom_completion: None, }, None, ) } else { - let name = working_set.get_span_contents(span).to_vec(); - // this seems okay to set it to unknown here, but we should double-check - let id = working_set.add_variable(name, Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span, - ty: Type::Unknown, - }, - None, - ) + (garbage(span), Some(ParseError::VariableNotFound(span))) } } else { (garbage(span), err) } } -pub fn parse_full_column_path( +pub fn parse_full_cell_path( working_set: &mut StateWorkingSet, + implicit_head: Option, span: Span, ) -> (Expression, Option) { // FIXME: assume for now a paren expr, but needs more - let full_column_span = span; + let full_cell_span = span; let source = working_set.get_span_contents(span); let mut error = None; let (tokens, err) = lex(source, span.start, &[b'\n'], &[b'.']); error = error.or(err); - let mut tokens = tokens.into_iter(); - if let Some(head) = tokens.next() { + let mut tokens = tokens.into_iter().peekable(); + if let Some(head) = tokens.peek() { let bytes = working_set.get_span_contents(head.span); - let head = if bytes.starts_with(b"(") { + let (head, mut expect_dot) = if bytes.starts_with(b"(") { let mut start = head.span.start; let mut end = head.span.end; @@ -1087,27 +1200,44 @@ pub fn parse_full_column_path( let source = working_set.get_span_contents(span); - let (tokens, err) = lex(source, span.start, &[b'\n'], &[]); + let (output, err) = lex(source, span.start, &[b'\n'], &[]); error = error.or(err); - let (output, err) = lite_parse(&tokens); + let (output, err) = lite_parse(&output); error = error.or(err); let (output, err) = parse_block(working_set, &output, true); error = error.or(err); let block_id = working_set.add_block(output); + tokens.next(); - Expression { - expr: Expr::Subexpression(block_id), - span, - ty: Type::Unknown, // FIXME - } + ( + Expression { + expr: Expr::Subexpression(block_id), + span, + ty: Type::Unknown, // FIXME + custom_completion: None, + }, + true, + ) } else if bytes.starts_with(b"$") { let (out, err) = parse_variable_expr(working_set, head.span); error = error.or(err); - out + tokens.next(); + + (out, true) + } else if let Some(var_id) = implicit_head { + ( + Expression { + expr: Expr::Var(var_id), + span: Span::unknown(), + ty: Type::Unknown, + custom_completion: None, + }, + false, + ) } else { return ( garbage(span), @@ -1121,7 +1251,6 @@ pub fn parse_full_column_path( let mut tail = vec![]; - let mut expect_dot = true; for path_element in tokens { let bytes = working_set.get_span_contents(path_element.span); @@ -1171,7 +1300,8 @@ pub fn parse_full_column_path( Expression { expr: Expr::FullCellPath(Box::new(FullCellPath { head, tail })), ty: Type::Unknown, - span: full_column_span, + span: full_cell_span, + custom_completion: None, }, error, ) @@ -1199,6 +1329,7 @@ pub fn parse_string( expr: Expr::String(token), span, ty: Type::String, + custom_completion: None, }, None, ) @@ -1225,7 +1356,7 @@ pub fn parse_shape_name( b"int" => SyntaxShape::Int, b"path" => SyntaxShape::FilePath, b"glob" => SyntaxShape::GlobPattern, - b"block" => SyntaxShape::Block, + b"block" => SyntaxShape::Block(None), //FIXME b"cond" => SyntaxShape::RowCondition, b"operator" => SyntaxShape::Operator, b"math" => SyntaxShape::MathExpression, @@ -1246,6 +1377,95 @@ pub fn parse_type(_working_set: &StateWorkingSet, bytes: &[u8]) -> Type { } } +pub fn parse_import_pattern( + working_set: &mut StateWorkingSet, + span: Span, +) -> (ImportPattern, Option) { + let source = working_set.get_span_contents(span); + let mut error = None; + + let (tokens, err) = lex(source, span.start, &[], &[b'.']); + error = error.or(err); + + if tokens.is_empty() { + return ( + ImportPattern { + head: vec![], + members: vec![], + }, + Some(ParseError::MissingImportPattern(span)), + ); + } + + let head = working_set.get_span_contents(tokens[0].span).to_vec(); + + if let Some(tail) = tokens.get(2) { + // FIXME: expand this to handle deeper imports once we support module imports + let tail_span = tail.span; + let tail = working_set.get_span_contents(tail.span); + if tail == b"*" { + ( + ImportPattern { + head, + members: vec![ImportPatternMember::Glob { span: tail_span }], + }, + error, + ) + } else if tail.starts_with(b"[") { + let (result, err) = parse_list_expression(working_set, tail_span, &SyntaxShape::String); + error = error.or(err); + + let mut output = vec![]; + + match result { + Expression { + expr: Expr::List(list), + .. + } => { + for l in list { + let contents = working_set.get_span_contents(l.span); + output.push((contents.to_vec(), l.span)); + } + + ( + ImportPattern { + head, + members: vec![ImportPatternMember::List { names: output }], + }, + error, + ) + } + _ => ( + ImportPattern { + head, + members: vec![], + }, + Some(ParseError::ExportNotFound(result.span)), + ), + } + } else { + ( + ImportPattern { + head, + members: vec![ImportPatternMember::Name { + name: tail.to_vec(), + span: tail_span, + }], + }, + error, + ) + } + } else { + ( + ImportPattern { + head, + members: vec![], + }, + None, + ) + } +} + pub fn parse_var_with_opt_type( working_set: &mut StateWorkingSet, spans: &[Span], @@ -1268,6 +1488,7 @@ pub fn parse_var_with_opt_type( expr: Expr::Var(id), span: span(&spans[*spans_idx - 1..*spans_idx + 1]), ty, + custom_completion: None, }, None, ) @@ -1278,6 +1499,7 @@ pub fn parse_var_with_opt_type( expr: Expr::Var(id), span: spans[*spans_idx], ty: Type::Unknown, + custom_completion: None, }, Some(ParseError::MissingType(spans[*spans_idx])), ) @@ -1290,16 +1512,47 @@ pub fn parse_var_with_opt_type( expr: Expr::Var(id), span: span(&spans[*spans_idx..*spans_idx + 1]), ty: Type::Unknown, + custom_completion: None, }, None, ) } } + +pub fn expand_to_cell_path( + working_set: &mut StateWorkingSet, + expression: &mut Expression, + var_id: VarId, +) { + if let Expression { + expr: Expr::String(_), + span, + .. + } = expression + { + // Re-parse the string as if it were a cell-path + let (new_expression, _err) = parse_full_cell_path(working_set, Some(var_id), *span); + + *expression = new_expression; + } +} + pub fn parse_row_condition( working_set: &mut StateWorkingSet, spans: &[Span], ) -> (Expression, Option) { - parse_math_expression(working_set, spans) + let var_id = working_set.add_variable(b"$it".to_vec(), Type::Unknown); + let (expression, err) = parse_math_expression(working_set, spans, Some(var_id)); + let span = span(spans); + ( + Expression { + ty: Type::Bool, + span, + expr: Expr::RowCondition(var_id, Box::new(expression)), + custom_completion: None, + }, + err, + ) } pub fn parse_signature( @@ -1337,6 +1590,7 @@ pub fn parse_signature( expr: Expr::Signature(sig), span, ty: Type::Unknown, + custom_completion: None, }, error, ) @@ -1736,6 +1990,7 @@ pub fn parse_list_expression( } else { Type::Unknown })), + custom_completion: None, }, error, ) @@ -1743,13 +1998,13 @@ pub fn parse_list_expression( pub fn parse_table_expression( working_set: &mut StateWorkingSet, - span: Span, + original_span: Span, ) -> (Expression, Option) { - let bytes = working_set.get_span_contents(span); + let bytes = working_set.get_span_contents(original_span); let mut error = None; - let mut start = span.start; - let mut end = span.end; + let mut start = original_span.start; + let mut end = original_span.end; if bytes.starts_with(b"[") { start += 1; @@ -1784,12 +2039,13 @@ pub fn parse_table_expression( expr: Expr::List(vec![]), span, ty: Type::List(Box::new(Type::Unknown)), + custom_completion: None, }, None, ), 1 => { // List - parse_list_expression(working_set, span, &SyntaxShape::Any) + parse_list_expression(working_set, original_span, &SyntaxShape::Any) } _ => { let mut table_headers = vec![]; @@ -1819,9 +2075,27 @@ pub fn parse_table_expression( error = error.or(err); if let Expression { expr: Expr::List(values), + span, .. } = values { + match values.len().cmp(&table_headers.len()) { + std::cmp::Ordering::Less => { + error = error.or_else(|| { + Some(ParseError::MissingColumns(table_headers.len(), span)) + }) + } + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Greater => { + error = error.or_else(|| { + Some(ParseError::ExtraColumns( + table_headers.len(), + values[table_headers.len()].span, + )) + }) + } + } + rows.push(values); } } @@ -1830,7 +2104,8 @@ pub fn parse_table_expression( Expression { expr: Expr::Table(table_headers, rows), span, - ty: Type::List(Box::new(Type::Unknown)), + ty: Type::Table, + custom_completion: None, }, error, ) @@ -1840,6 +2115,7 @@ pub fn parse_table_expression( pub fn parse_block_expression( working_set: &mut StateWorkingSet, + shape: &SyntaxShape, span: Span, ) -> (Expression, Option) { let bytes = working_set.get_span_contents(span); @@ -1880,7 +2156,7 @@ pub fn parse_block_expression( working_set.enter_scope(); // Check to see if we have parameters - let (signature, amt_to_skip): (Option>, usize) = match output.first() { + let (mut signature, amt_to_skip): (Option>, usize) = match output.first() { Some(Token { contents: TokenContents::Pipe, span, @@ -1926,12 +2202,31 @@ pub fn parse_block_expression( let (output, err) = lite_parse(&output[amt_to_skip..]); error = error.or(err); + if let SyntaxShape::Block(Some(v)) = shape { + if signature.is_none() && v.len() == 1 { + // We'll assume there's an `$it` present + let var_id = working_set.add_variable(b"$it".to_vec(), Type::Unknown); + + let mut new_sigature = Signature::new(""); + new_sigature.required_positional.push(PositionalArg { + var_id: Some(var_id), + name: "$it".into(), + desc: String::new(), + shape: SyntaxShape::Any, + }); + + signature = Some(Box::new(new_sigature)); + } + } + let (mut output, err) = parse_block(working_set, &output, false); error = error.or(err); if let Some(signature) = signature { output.signature = signature; } else if let Some(last) = working_set.delta.scope.last() { + // FIXME: this only supports the top $it. Instead, we should look for a free $it in the expression. + if let Some(var_id) = last.get_var(b"$it") { let mut signature = Signature::new(""); signature.required_positional.push(PositionalArg { @@ -1953,6 +2248,7 @@ pub fn parse_block_expression( expr: Expr::Block(block_id), span, ty: Type::Block, + custom_completion: None, }, error, ) @@ -1979,11 +2275,11 @@ pub fn parse_value( if let (expr, None) = parse_range(working_set, span) { return (expr, None); } else { - return parse_full_column_path(working_set, span); + return parse_full_cell_path(working_set, None, span); } } else if bytes.starts_with(b"{") { - if matches!(shape, SyntaxShape::Block) || matches!(shape, SyntaxShape::Any) { - return parse_block_expression(working_set, span); + if matches!(shape, SyntaxShape::Block(_)) || matches!(shape, SyntaxShape::Any) { + return parse_block_expression(working_set, shape, span); } else { return ( Expression::garbage(span), @@ -2006,15 +2302,20 @@ pub fn parse_value( } match shape { + SyntaxShape::Custom(shape, custom_completion) => { + let (mut expression, err) = parse_value(working_set, span, shape); + expression.custom_completion = Some(custom_completion.clone()); + (expression, err) + } SyntaxShape::Number => parse_number(bytes, span), SyntaxShape::Int => parse_int(bytes, span), SyntaxShape::Range => parse_range(working_set, span), SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { parse_string(working_set, span) } - SyntaxShape::Block => { + SyntaxShape::Block(_) => { if bytes.starts_with(b"{") { - parse_block_expression(working_set, span) + parse_block_expression(working_set, shape, span) } else { ( Expression::garbage(span), @@ -2054,17 +2355,7 @@ pub fn parse_value( } SyntaxShape::Any => { if bytes.starts_with(b"[") { - let shapes = [SyntaxShape::Table]; - for shape in shapes.iter() { - if let (s, None) = parse_value(working_set, span, shape) { - return (s, None); - } - } - parse_value( - working_set, - span, - &SyntaxShape::List(Box::new(SyntaxShape::Any)), - ) + parse_value(working_set, span, &SyntaxShape::Table) } else { let shapes = [ SyntaxShape::Int, @@ -2072,9 +2363,7 @@ pub fn parse_value( SyntaxShape::Range, SyntaxShape::Filesize, SyntaxShape::Duration, - SyntaxShape::Block, - SyntaxShape::Table, - SyntaxShape::List(Box::new(SyntaxShape::Any)), + SyntaxShape::Block(None), SyntaxShape::String, ]; for shape in shapes.iter() { @@ -2130,6 +2419,7 @@ pub fn parse_operator( expr: Expr::Operator(operator), span, ty: Type::Unknown, + custom_completion: None, }, None, ) @@ -2138,6 +2428,7 @@ pub fn parse_operator( pub fn parse_math_expression( working_set: &mut StateWorkingSet, spans: &[Span], + lhs_row_var_id: Option, ) -> (Expression, Option) { // As the expr_stack grows, we increase the required precedence to grow larger // If, at any time, the operator we're looking at is the same or lower precedence @@ -2196,6 +2487,10 @@ pub fn parse_math_expression( .pop() .expect("internal error: expression stack empty"); + if let Some(row_var_id) = lhs_row_var_id { + expand_to_cell_path(working_set, &mut lhs, row_var_id); + } + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); error = error.or(err); @@ -2204,6 +2499,7 @@ pub fn parse_math_expression( expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), span: op_span, ty: result_ty, + custom_completion: None, }); } } @@ -2226,6 +2522,10 @@ pub fn parse_math_expression( .pop() .expect("internal error: expression stack empty"); + if let Some(row_var_id) = lhs_row_var_id { + expand_to_cell_path(working_set, &mut lhs, row_var_id); + } + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); error = error.or(err); @@ -2234,6 +2534,7 @@ pub fn parse_math_expression( expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), span: binary_op_span, ty: result_ty, + custom_completion: None, }); } @@ -2252,7 +2553,7 @@ pub fn parse_expression( match bytes[0] { b'0' | b'1' | b'2' | b'3' | b'4' | b'5' | b'6' | b'7' | b'8' | b'9' | b'(' | b'{' - | b'[' | b'$' | b'"' | b'\'' | b'-' => parse_math_expression(working_set, spans), + | b'[' | b'$' | b'"' | b'\'' | b'-' => parse_math_expression(working_set, spans, None), _ => parse_call(working_set, spans, true), } } @@ -2274,306 +2575,22 @@ pub fn parse_variable( } } -pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) { - let name = working_set.get_span_contents(spans[0]); - - if name == b"def" && spans.len() >= 4 { - let (name_expr, ..) = parse_string(working_set, spans[1]); - let name = name_expr.as_string(); - - working_set.enter_scope(); - // FIXME: because parse_signature will update the scope with the variables it sees - // we end up parsing the signature twice per def. The first time is during the predecl - // so that we can see the types that are part of the signature, which we need for parsing. - // The second time is when we actually parse the body itworking_set. - // We can't reuse the first time because the variables that are created during parse_signature - // are lost when we exit the scope below. - let (sig, ..) = parse_signature(working_set, spans[2]); - let signature = sig.as_signature(); - working_set.exit_scope(); - - if let (Some(name), Some(mut signature)) = (name, signature) { - signature.name = name; - let decl = signature.predeclare(); - - working_set.add_decl(decl); - } - } -} - -/// Parser for the def command -pub fn parse_def( - working_set: &mut StateWorkingSet, - spans: &[Span], -) -> (Statement, Option) { - let mut error = None; - let name = working_set.get_span_contents(spans[0]); - - if name == b"def" && spans.len() >= 4 { - //FIXME: don't use expect here - let (name_expr, err) = parse_string(working_set, spans[1]); - error = error.or(err); - - working_set.enter_scope(); - let (sig, err) = parse_signature(working_set, spans[2]); - error = error.or(err); - - let (block, err) = parse_block_expression(working_set, spans[3]); - error = error.or(err); - working_set.exit_scope(); - - let name = name_expr.as_string(); - - let signature = sig.as_signature(); - - let block_id = block.as_block(); - - match (name, signature, block_id) { - (Some(name), Some(mut signature), Some(block_id)) => { - let decl_id = working_set - .find_decl(name.as_bytes()) - .expect("internal error: predeclaration failed to add definition"); - - let declaration = working_set.get_decl_mut(decl_id); - - signature.name = name; - - *declaration = signature.into_block_command(block_id); - - let def_decl_id = working_set - .find_decl(b"def") - .expect("internal error: missing def command"); - - let call = Box::new(Call { - head: spans[0], - decl_id: def_decl_id, - positional: vec![name_expr, sig, block], - named: vec![], - }); - - ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Call(call), - span: span(spans), - ty: Type::Unknown, - }])), - error, - ) - } - _ => ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }])), - error, - ), - } - } else { - ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }])), - Some(ParseError::UnknownState( - "internal error: definition unparseable".into(), - span(spans), - )), - ) - } -} - -/// Parser for the alias command -pub fn parse_alias( - working_set: &mut StateWorkingSet, - spans: &[Span], -) -> (Statement, Option) { - let name = working_set.get_span_contents(spans[0]); - - if name == b"alias" { - if let Some(decl_id) = working_set.find_decl(b"alias") { - let (call, call_span, _) = - parse_internal_call(working_set, spans[0], &spans[1..], decl_id); - - if spans.len() >= 4 { - let alias_name = working_set.get_span_contents(spans[1]); - - let alias_name = if alias_name.starts_with(b"\"") - && alias_name.ends_with(b"\"") - && alias_name.len() > 1 - { - alias_name[1..(alias_name.len() - 1)].to_vec() - } else { - alias_name.to_vec() - }; - let _equals = working_set.get_span_contents(spans[2]); - - let replacement = spans[3..].to_vec(); - - //println!("{:?} {:?}", alias_name, replacement); - - working_set.add_alias(alias_name, replacement); - } - - return ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }])), - None, - ); - } - } - - ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }])), - Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), - span(spans), - )), - ) -} - -/// Parse let command -pub fn parse_let( - working_set: &mut StateWorkingSet, - spans: &[Span], -) -> (Statement, Option) { - let name = working_set.get_span_contents(spans[0]); - - if name == b"let" { - if let Some(decl_id) = working_set.find_decl(b"let") { - let (call, call_span, err) = - parse_internal_call(working_set, spans[0], &spans[1..], decl_id); - - // Update the variable to the known type if we can. - if err.is_none() { - let var_id = call.positional[0] - .as_var() - .expect("internal error: expected variable"); - let rhs_type = call.positional[1].ty.clone(); - - working_set.set_variable_type(var_id, rhs_type); - } - - return ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }])), - err, - ); - } - } - ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }])), - Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), - span(spans), - )), - ) -} - -/// Parser for the source command -pub fn parse_source( - working_set: &mut StateWorkingSet, - spans: &[Span], -) -> (Statement, Option) { - let name = working_set.get_span_contents(spans[0]); - - if name == b"source" { - if let Some(decl_id) = working_set.find_decl(b"source") { - let (call, call_span, _) = - parse_internal_call(working_set, spans[0], &spans[1..], decl_id); - - // Command and one file name - if spans.len() >= 2 { - let name_expr = working_set.get_span_contents(spans[1]); - if let Ok(filename) = String::from_utf8(name_expr.to_vec()) { - let source_file = Path::new(&filename); - - let path = source_file; - let contents = std::fs::read(path); - - if let Ok(contents) = contents { - let (block, err) = parse( - working_set, - path.file_name().and_then(|x| x.to_str()), - &contents, - true, - ); - if let None = err { - // Successful parse - // What should I be doing here? - let block_id = working_set.add_block(block); - // return ( - // // Successful parse - // // Why creating a pipeline here for only one expression? - // // Is there a way to only make this a declaration? - // Statement::Pipeline(Pipeline::from_vec(vec![Expression { - // expr: Expr::Subexpression(block_id), - // span: span(spans), - // ty: Type::Unknown, // FIXME - // }])), - // None, - // ); - } - } - } - } - - return ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }])), - None, - ); - } - } - - ( - Statement::Pipeline(Pipeline::from_vec(vec![Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }])), - Some(ParseError::UnknownState( - "internal error: source statement unparseable".into(), - span(spans), - )), - ) -} - -/// Parse a statement. Check if def, let, alias, or source command can process it properly pub fn parse_statement( working_set: &mut StateWorkingSet, spans: &[Span], ) -> (Statement, Option) { - // FIXME: improve errors by checking keyword first - if let (decl, None) = parse_def(working_set, spans) { - (decl, None) - } else if let (stmt, None) = parse_let(working_set, spans) { - (stmt, None) - } else if let (stmt, None) = parse_alias(working_set, spans) { - (stmt, None) - } else if let (decl, None) = parse_source(working_set, spans) { - (decl, None) - } else { - let (expr, err) = parse_expression(working_set, spans); - (Statement::Pipeline(Pipeline::from_vec(vec![expr])), err) + let name = working_set.get_span_contents(spans[0]); + + match name { + b"def" => parse_def(working_set, spans), + b"let" => parse_let(working_set, spans), + b"alias" => parse_alias(working_set, spans), + b"module" => parse_module(working_set, spans), + b"use" => parse_use(working_set, spans), + _ => { + let (expr, err) = parse_expression(working_set, spans); + (Statement::Pipeline(Pipeline::from_vec(vec![expr])), err) + } } } @@ -2582,13 +2599,10 @@ pub fn parse_block( lite_block: &LiteBlock, scoped: bool, ) -> (Block, Option) { - let mut error = None; if scoped { working_set.enter_scope(); } - let mut block = Block::new(); - // Pre-declare any definition so that definitions // that share the same block can see each other for pipeline in &lite_block.block { @@ -2597,25 +2611,41 @@ pub fn parse_block( } } - for pipeline in &lite_block.block { - if pipeline.commands.len() > 1 { - let mut output = vec![]; - for command in &pipeline.commands { - let (expr, err) = parse_expression(working_set, &command.parts); - error = error.or(err); + let mut error = None; - output.push(expr); + let block: Block = lite_block + .block + .iter() + .map(|pipeline| { + if pipeline.commands.len() > 1 { + let output = pipeline + .commands + .iter() + .map(|command| { + let (expr, err) = parse_expression(working_set, &command.parts); + + if error.is_none() { + error = err; + } + + expr + }) + .collect::>(); + + Statement::Pipeline(Pipeline { + expressions: output, + }) + } else { + let (stmt, err) = parse_statement(working_set, &pipeline.commands[0].parts); + + if error.is_none() { + error = err; + } + + stmt } - block.stmts.push(Statement::Pipeline(Pipeline { - expressions: output, - })); - } else { - let (stmt, err) = parse_statement(working_set, &pipeline.commands[0].parts); - error = error.or(err); - - block.stmts.push(stmt); - } - } + }) + .into(); if scoped { working_set.exit_scope(); diff --git a/crates/nu-parser/src/type_check.rs b/crates/nu-parser/src/type_check.rs index 67a082c67f..fa07f755da 100644 --- a/crates/nu-parser/src/type_check.rs +++ b/crates/nu-parser/src/type_check.rs @@ -20,6 +20,7 @@ pub fn math_result_type( op: &mut Expression, rhs: &mut Expression, ) -> (Type, Option) { + //println!("checking: {:?} {:?} {:?}", lhs, op, rhs); match &op.expr { Expr::Operator(operator) => match operator { Operator::Plus => match (&lhs.ty, &rhs.ty) { @@ -31,6 +32,7 @@ pub fn math_result_type( (Type::Unknown, _) => (Type::Unknown, None), (_, Type::Unknown) => (Type::Unknown, None), (Type::Int, _) => { + let ty = rhs.ty.clone(); *rhs = Expression::garbage(rhs.span); ( Type::Unknown, @@ -39,7 +41,7 @@ pub fn math_result_type( lhs.span, lhs.ty.clone(), rhs.span, - rhs.ty.clone(), + ty, )), ) } diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index ef52f1fbcf..26393f37e0 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -2,10 +2,43 @@ use nu_parser::ParseError; use nu_parser::*; use nu_protocol::{ ast::{Expr, Expression, Pipeline, Statement}, - engine::{EngineState, StateWorkingSet}, + engine::{Command, EngineState, StateWorkingSet}, Signature, SyntaxShape, }; +#[cfg(test)] +pub struct Let; + +#[cfg(test)] +impl Command for Let { + fn name(&self) -> &str { + "let" + } + + fn usage(&self) -> &str { + "Create a variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("let") + .required("var_name", SyntaxShape::VarWithOptType, "variable name") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", + ) + } + + fn run( + &self, + _context: &nu_protocol::engine::EvaluationContext, + _call: &nu_protocol::ast::Call, + _input: nu_protocol::Value, + ) -> Result { + todo!() + } +} + #[test] pub fn parse_int() { let engine_state = EngineState::new(); @@ -164,6 +197,7 @@ mod range { Expression { expr: Expr::Range( Some(_), + None, Some(_), RangeOperator { inclusion: RangeInclusion::Inclusive, @@ -195,6 +229,7 @@ mod range { Expression { expr: Expr::Range( Some(_), + None, Some(_), RangeOperator { inclusion: RangeInclusion::RightExclusive, @@ -209,6 +244,38 @@ mod range { } } + #[test] + fn parse_reverse_range() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + let (block, err) = parse(&mut working_set, None, b"10..0", true); + + assert!(err.is_none()); + assert!(block.len() == 1); + match &block[0] { + Statement::Pipeline(Pipeline { expressions }) => { + assert!(expressions.len() == 1); + assert!(matches!( + expressions[0], + Expression { + expr: Expr::Range( + Some(_), + None, + Some(_), + RangeOperator { + inclusion: RangeInclusion::Inclusive, + .. + } + ), + .. + } + )) + } + _ => panic!("No match"), + } + } + #[test] fn parse_subexpression_range() { let engine_state = EngineState::new(); @@ -226,6 +293,7 @@ mod range { Expression { expr: Expr::Range( Some(_), + None, Some(_), RangeOperator { inclusion: RangeInclusion::RightExclusive, @@ -245,6 +313,8 @@ mod range { let engine_state = EngineState::new(); let mut working_set = StateWorkingSet::new(&engine_state); + working_set.add_decl(Box::new(Let)); + let (block, err) = parse(&mut working_set, None, b"let a = 2; $a..10", true); assert!(err.is_none()); @@ -257,6 +327,7 @@ mod range { Expression { expr: Expr::Range( Some(_), + None, Some(_), RangeOperator { inclusion: RangeInclusion::Inclusive, @@ -276,6 +347,8 @@ mod range { let engine_state = EngineState::new(); let mut working_set = StateWorkingSet::new(&engine_state); + working_set.add_decl(Box::new(Let)); + let (block, err) = parse(&mut working_set, None, b"let a = 2; $a..<($a + 10)", true); assert!(err.is_none()); @@ -288,6 +361,7 @@ mod range { Expression { expr: Expr::Range( Some(_), + None, Some(_), RangeOperator { inclusion: RangeInclusion::RightExclusive, @@ -320,6 +394,39 @@ mod range { expr: Expr::Range( Some(_), None, + None, + RangeOperator { + inclusion: RangeInclusion::Inclusive, + .. + } + ), + .. + } + )) + } + _ => panic!("No match"), + } + } + + #[test] + fn parse_left_unbounded_range() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + let (block, err) = parse(&mut working_set, None, b"..10", true); + + assert!(err.is_none()); + assert!(block.len() == 1); + match &block[0] { + Statement::Pipeline(Pipeline { expressions }) => { + assert!(expressions.len() == 1); + assert!(matches!( + expressions[0], + Expression { + expr: Expr::Range( + None, + None, + Some(_), RangeOperator { inclusion: RangeInclusion::Inclusive, .. @@ -349,6 +456,39 @@ mod range { expressions[0], Expression { expr: Expr::Range( + Some(_), + None, + Some(_), + RangeOperator { + inclusion: RangeInclusion::Inclusive, + .. + } + ), + .. + } + )) + } + _ => panic!("No match"), + } + } + + #[test] + fn parse_float_range() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + let (block, err) = parse(&mut working_set, None, b"2.0..4.0..10.0", true); + + assert!(err.is_none()); + assert!(block.len() == 1); + match &block[0] { + Statement::Pipeline(Pipeline { expressions }) => { + assert!(expressions.len() == 1); + assert!(matches!( + expressions[0], + Expression { + expr: Expr::Range( + Some(_), Some(_), Some(_), RangeOperator { diff --git a/crates/nu-path/Cargo.toml b/crates/nu-path/Cargo.toml new file mode 100644 index 0000000000..e2fc2b1ab9 --- /dev/null +++ b/crates/nu-path/Cargo.toml @@ -0,0 +1,12 @@ +[package] +authors = ["The Nu Project Contributors"] +description = "Path handling library for Nushell" +edition = "2018" +license = "MIT" +name = "nu-path" +version = "0.37.1" + +[dependencies] +dirs-next = "2.0.0" +dunce = "1.0.1" + diff --git a/crates/nu-path/README.md b/crates/nu-path/README.md new file mode 100644 index 0000000000..382fd687bd --- /dev/null +++ b/crates/nu-path/README.md @@ -0,0 +1,3 @@ +# nu-path + +This crate takes care of path handling in Nushell, such as canonicalization and component expansion, as well as other path-related utilities. diff --git a/crates/nu-path/src/dots.rs b/crates/nu-path/src/dots.rs new file mode 100644 index 0000000000..b6025c479c --- /dev/null +++ b/crates/nu-path/src/dots.rs @@ -0,0 +1,259 @@ +use std::path::{is_separator, Component, Path, PathBuf}; + +const EXPAND_STR: &str = if cfg!(windows) { r"..\" } else { "../" }; + +fn handle_dots_push(string: &mut String, count: u8) { + if count < 1 { + return; + } + + if count == 1 { + string.push('.'); + return; + } + + for _ in 0..(count - 1) { + string.push_str(EXPAND_STR); + } + + string.pop(); // remove last '/' +} + +/// Expands any occurence of more than two dots into a sequence of ../ (or ..\ on windows), e.g., +/// "..." into "../..", "...." into "../../../", etc. +pub fn expand_ndots(path: impl AsRef) -> PathBuf { + // Check if path is valid UTF-8 and if not, return it as it is to avoid breaking it via string + // conversion. + let path_str = match path.as_ref().to_str() { + Some(s) => s, + None => return path.as_ref().into(), + }; + + // find if we need to expand any >2 dot paths and early exit if not + let mut dots_count = 0u8; + let ndots_present = { + for chr in path_str.chars() { + if chr == '.' { + dots_count += 1; + } else { + if is_separator(chr) && (dots_count > 2) { + // this path component had >2 dots + break; + } + + dots_count = 0; + } + } + + dots_count > 2 + }; + + if !ndots_present { + return path.as_ref().into(); + } + + let mut dots_count = 0u8; + let mut expanded = String::new(); + for chr in path_str.chars() { + if chr == '.' { + dots_count += 1; + } else { + if is_separator(chr) { + // check for dots expansion only at path component boundaries + handle_dots_push(&mut expanded, dots_count); + dots_count = 0; + } else { + // got non-dot within path component => do not expand any dots + while dots_count > 0 { + expanded.push('.'); + dots_count -= 1; + } + } + expanded.push(chr); + } + } + + handle_dots_push(&mut expanded, dots_count); + + expanded.into() +} + +/// Expand "." and ".." into nothing and parent directory, respectively. +pub fn expand_dots(path: impl AsRef) -> PathBuf { + let path = path.as_ref(); + + // Early-exit if path does not contain '.' or '..' + if !path + .components() + .any(|c| std::matches!(c, Component::CurDir | Component::ParentDir)) + { + return path.into(); + } + + let mut result = PathBuf::with_capacity(path.as_os_str().len()); + + // Only pop/skip path elements if the previous one was an actual path element + let prev_is_normal = |p: &Path| -> bool { + p.components() + .next_back() + .map(|c| std::matches!(c, Component::Normal(_))) + .unwrap_or(false) + }; + + path.components().for_each(|component| match component { + Component::ParentDir if prev_is_normal(&result) => { + result.pop(); + } + Component::CurDir if prev_is_normal(&result) => {} + _ => result.push(component), + }); + + dunce::simplified(&result).to_path_buf() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expand_two_dots() { + let path = Path::new("/foo/bar/.."); + + assert_eq!( + PathBuf::from("/foo"), // missing path + expand_dots(path) + ); + } + + #[test] + fn expand_dots_with_curdir() { + let path = Path::new("/foo/./bar/./baz"); + + assert_eq!(PathBuf::from("/foo/bar/baz"), expand_dots(path)); + } + + fn check_ndots_expansion(expected: &str, s: &str) { + let expanded = expand_ndots(Path::new(s)); + assert_eq!(Path::new(expected), &expanded); + } + + // common tests + #[test] + fn string_without_ndots() { + check_ndots_expansion("../hola", "../hola"); + } + + #[test] + fn string_with_three_ndots_and_chars() { + check_ndots_expansion("a...b", "a...b"); + } + + #[test] + fn string_with_two_ndots_and_chars() { + check_ndots_expansion("a..b", "a..b"); + } + + #[test] + fn string_with_one_dot_and_chars() { + check_ndots_expansion("a.b", "a.b"); + } + + #[test] + fn expand_dots_double_dots_no_change() { + // Can't resolve this as we don't know our parent dir + assert_eq!(Path::new(".."), expand_dots(Path::new(".."))); + } + + #[test] + fn expand_dots_single_dot_no_change() { + // Can't resolve this as we don't know our current dir + assert_eq!(Path::new("."), expand_dots(Path::new("."))); + } + + #[test] + fn expand_dots_multi_single_dots_no_change() { + assert_eq!(Path::new("././."), expand_dots(Path::new("././."))); + } + + #[test] + fn expand_multi_double_dots_no_change() { + assert_eq!(Path::new("../../../"), expand_dots(Path::new("../../../"))); + } + + #[test] + fn expand_dots_no_change_with_dirs() { + // Can't resolve this as we don't know our parent dir + assert_eq!( + Path::new("../../../dir1/dir2/"), + expand_dots(Path::new("../../../dir1/dir2")) + ); + } + + #[test] + fn expand_dots_simple() { + assert_eq!(Path::new("/foo"), expand_dots(Path::new("/foo/bar/.."))); + } + + #[test] + fn expand_dots_complex() { + assert_eq!( + Path::new("/test"), + expand_dots(Path::new("/foo/./bar/../../test/././test2/../")) + ); + } + + #[cfg(windows)] + mod windows { + use super::*; + + #[test] + fn string_with_three_ndots() { + check_ndots_expansion(r"..\..", "..."); + } + + #[test] + fn string_with_mixed_ndots_and_chars() { + check_ndots_expansion( + r"a...b/./c..d/../e.f/..\..\..//.", + "a...b/./c..d/../e.f/....//.", + ); + } + + #[test] + fn string_with_three_ndots_and_final_slash() { + check_ndots_expansion(r"..\../", ".../"); + } + + #[test] + fn string_with_three_ndots_and_garbage() { + check_ndots_expansion(r"ls ..\../ garbage.*[", "ls .../ garbage.*["); + } + } + + #[cfg(not(windows))] + mod non_windows { + use super::*; + #[test] + fn string_with_three_ndots() { + check_ndots_expansion(r"../..", "..."); + } + + #[test] + fn string_with_mixed_ndots_and_chars() { + check_ndots_expansion( + "a...b/./c..d/../e.f/../../..//.", + "a...b/./c..d/../e.f/....//.", + ); + } + + #[test] + fn string_with_three_ndots_and_final_slash() { + check_ndots_expansion("../../", ".../"); + } + + #[test] + fn string_with_three_ndots_and_garbage() { + check_ndots_expansion("ls ../../ garbage.*[", "ls .../ garbage.*["); + } + } +} diff --git a/crates/nu-path/src/expansions.rs b/crates/nu-path/src/expansions.rs new file mode 100644 index 0000000000..3393a5793f --- /dev/null +++ b/crates/nu-path/src/expansions.rs @@ -0,0 +1,75 @@ +use std::io; +use std::path::{Path, PathBuf}; + +use super::dots::{expand_dots, expand_ndots}; +use super::tilde::expand_tilde; + +// Join a path relative to another path. Paths starting with tilde are considered as absolute. +fn join_path_relative(path: P, relative_to: Q) -> PathBuf +where + P: AsRef, + Q: AsRef, +{ + let path = path.as_ref(); + let relative_to = relative_to.as_ref(); + + if path == Path::new(".") { + // Joining a Path with '.' appends a '.' at the end, making the prompt + // more ugly - so we don't do anything, which should result in an equal + // path on all supported systems. + relative_to.into() + } else if path.starts_with("~") { + // do not end up with "/some/path/~" + path.into() + } else { + relative_to.join(path) + } +} + +/// Resolve all symbolic links and all components (tilde, ., .., ...+) and return the path in its +/// absolute form. +/// +/// Fails under the same conditions as +/// [std::fs::canonicalize](https://doc.rust-lang.org/std/fs/fn.canonicalize.html). +pub fn canonicalize(path: impl AsRef) -> io::Result { + let path = expand_tilde(path); + let path = expand_ndots(path); + + dunce::canonicalize(path) +} + +/// Same as canonicalize() but the input path is specified relative to another path +pub fn canonicalize_with(path: P, relative_to: Q) -> io::Result +where + P: AsRef, + Q: AsRef, +{ + let path = join_path_relative(path, relative_to); + + canonicalize(path) +} + +/// Resolve only path components (tilde, ., .., ...+), if possible. +/// +/// The function works in a "best effort" mode: It does not fail but rather returns the unexpanded +/// version if the expansion is not possible. +/// +/// Furthermore, unlike canonicalize(), it does not use sys calls (such as readlink). +/// +/// Does not convert to absolute form nor does it resolve symlinks. +pub fn expand_path(path: impl AsRef) -> PathBuf { + let path = expand_tilde(path); + let path = expand_ndots(path); + expand_dots(path) +} + +/// Same as expand_path() but the input path is specified relative to another path +pub fn expand_path_with(path: P, relative_to: Q) -> PathBuf +where + P: AsRef, + Q: AsRef, +{ + let path = join_path_relative(path, relative_to); + + expand_path(path) +} diff --git a/crates/nu-path/src/lib.rs b/crates/nu-path/src/lib.rs new file mode 100644 index 0000000000..9606bc15cf --- /dev/null +++ b/crates/nu-path/src/lib.rs @@ -0,0 +1,8 @@ +mod dots; +mod expansions; +mod tilde; +mod util; + +pub use expansions::{canonicalize, canonicalize_with, expand_path, expand_path_with}; +pub use tilde::expand_tilde; +pub use util::trim_trailing_slash; diff --git a/crates/nu-path/src/tilde.rs b/crates/nu-path/src/tilde.rs new file mode 100644 index 0000000000..e1c7ec56a3 --- /dev/null +++ b/crates/nu-path/src/tilde.rs @@ -0,0 +1,85 @@ +use std::path::{Path, PathBuf}; + +fn expand_tilde_with(path: impl AsRef, home: Option) -> PathBuf { + let path = path.as_ref(); + + if !path.starts_with("~") { + return path.into(); + } + + match home { + None => path.into(), + Some(mut h) => { + if h == Path::new("/") { + // Corner case: `h` is a root directory; + // don't prepend extra `/`, just drop the tilde. + path.strip_prefix("~").unwrap_or(path).into() + } else { + if let Ok(p) = path.strip_prefix("~/") { + h.push(p) + } + h + } + } + } +} + +/// Expand tilde ("~") into a home directory if it is the first path component +pub fn expand_tilde(path: impl AsRef) -> PathBuf { + // TODO: Extend this to work with "~user" style of home paths + expand_tilde_with(path, dirs_next::home_dir()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_expanded(s: &str) { + let home = Path::new("/home"); + let buf = Some(PathBuf::from(home)); + assert!(expand_tilde_with(Path::new(s), buf).starts_with(&home)); + + // Tests the special case in expand_tilde for "/" as home + let home = Path::new("/"); + let buf = Some(PathBuf::from(home)); + assert!(!expand_tilde_with(Path::new(s), buf).starts_with("//")); + } + + fn check_not_expanded(s: &str) { + let home = PathBuf::from("/home"); + let expanded = expand_tilde_with(Path::new(s), Some(home)); + assert!(expanded == Path::new(s)); + } + + #[test] + fn string_with_tilde() { + check_expanded("~"); + } + + #[test] + fn string_with_tilde_forward_slash() { + check_expanded("~/test/"); + } + + #[test] + fn string_with_tilde_double_forward_slash() { + check_expanded("~//test/"); + } + + #[test] + fn does_not_expand_tilde_if_tilde_is_not_first_character() { + check_not_expanded("1~1"); + } + + #[cfg(windows)] + #[test] + fn string_with_tilde_backslash() { + check_expanded("~\\test/test2/test3"); + } + + #[cfg(windows)] + #[test] + fn string_with_double_tilde_backslash() { + check_expanded("~\\\\test\\test2/test3"); + } +} diff --git a/crates/nu-path/src/util.rs b/crates/nu-path/src/util.rs new file mode 100644 index 0000000000..63351e6aef --- /dev/null +++ b/crates/nu-path/src/util.rs @@ -0,0 +1,4 @@ +/// Trim trailing path separator from a string +pub fn trim_trailing_slash(s: &str) -> &str { + s.trim_end_matches(std::path::is_separator) +} diff --git a/crates/nu-path/tests/mod.rs b/crates/nu-path/tests/mod.rs new file mode 100644 index 0000000000..83c8c0aa0a --- /dev/null +++ b/crates/nu-path/tests/mod.rs @@ -0,0 +1 @@ +mod util; diff --git a/crates/nu-path/tests/util.rs b/crates/nu-path/tests/util.rs new file mode 100644 index 0000000000..601d9dd437 --- /dev/null +++ b/crates/nu-path/tests/util.rs @@ -0,0 +1,45 @@ +use nu_path::trim_trailing_slash; +use std::path::MAIN_SEPARATOR; + +/// Helper function that joins string literals with '/' or '\', based on the host OS +fn join_path_sep(pieces: &[&str]) -> String { + let sep_string = String::from(MAIN_SEPARATOR); + pieces.join(&sep_string) +} + +#[test] +fn trims_trailing_slash_without_trailing_slash() { + let path = join_path_sep(&["some", "path"]); + + let actual = trim_trailing_slash(&path); + + assert_eq!(actual, &path) +} + +#[test] +fn trims_trailing_slash() { + let path = join_path_sep(&["some", "path", ""]); + + let actual = trim_trailing_slash(&path); + let expected = join_path_sep(&["some", "path"]); + + assert_eq!(actual, &expected) +} + +#[test] +fn trims_many_trailing_slashes() { + let path = join_path_sep(&["some", "path", "", "", "", ""]); + + let actual = trim_trailing_slash(&path); + let expected = join_path_sep(&["some", "path"]); + + assert_eq!(actual, &expected) +} + +#[test] +fn trims_trailing_slash_empty() { + let path = String::from(MAIN_SEPARATOR); + let actual = trim_trailing_slash(&path); + + assert_eq!(actual, "") +} diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml index d6c800e68e..d2f3a7f915 100644 --- a/crates/nu-protocol/Cargo.toml +++ b/crates/nu-protocol/Cargo.toml @@ -6,4 +6,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -codespan-reporting = "0.11.1" \ No newline at end of file +thiserror = "1.0.29" +miette = "3.0.0" +serde = "1.0.130" \ No newline at end of file diff --git a/crates/nu-protocol/src/ast/block.rs b/crates/nu-protocol/src/ast/block.rs index d36e43d8ec..70273816b5 100644 --- a/crates/nu-protocol/src/ast/block.rs +++ b/crates/nu-protocol/src/ast/block.rs @@ -1,6 +1,6 @@ use std::ops::{Index, IndexMut}; -use crate::Signature; +use crate::{DeclId, Signature}; use super::Statement; @@ -8,6 +8,7 @@ use super::Statement; pub struct Block { pub signature: Box, pub stmts: Vec, + pub exports: Vec<(Vec, DeclId)>, // Assuming just defs for now } impl Block { @@ -45,6 +46,28 @@ impl Block { Self { signature: Box::new(Signature::new("")), stmts: vec![], + exports: vec![], + } + } + + pub fn with_exports(self, exports: Vec<(Vec, DeclId)>) -> Self { + Self { + signature: self.signature, + stmts: self.stmts, + exports, + } + } +} + +impl From for Block +where + T: Iterator, +{ + fn from(stmts: T) -> Self { + Self { + signature: Box::new(Signature::new("")), + stmts: stmts.collect(), + exports: vec![], } } } diff --git a/crates/nu-protocol/src/ast/call.rs b/crates/nu-protocol/src/ast/call.rs index efe5d7b482..c5ba21f68b 100644 --- a/crates/nu-protocol/src/ast/call.rs +++ b/crates/nu-protocol/src/ast/call.rs @@ -25,4 +25,24 @@ impl Call { named: vec![], } } + + pub fn has_flag(&self, flag_name: &str) -> bool { + for name in &self.named { + if flag_name == name.0 { + return true; + } + } + + false + } + + pub fn get_flag_expr(&self, flag_name: &str) -> Option { + for name in &self.named { + if flag_name == name.0 { + return name.1.clone(); + } + } + + None + } } diff --git a/crates/nu-protocol/src/ast/expr.rs b/crates/nu-protocol/src/ast/expr.rs index ac29cd24ba..62d7a25c8f 100644 --- a/crates/nu-protocol/src/ast/expr.rs +++ b/crates/nu-protocol/src/ast/expr.rs @@ -7,14 +7,16 @@ pub enum Expr { Int(i64), Float(f64), Range( - Option>, - Option>, + Option>, // from + Option>, // next value after "from" + Option>, // to RangeOperator, ), Var(VarId), Call(Box), - ExternalCall(Vec, Vec>), + ExternalCall(Span, Vec), Operator(Operator), + RowCondition(VarId, Box), BinaryOp(Box, Box, Box), //lhs, op, rhs Subexpression(BlockId), Block(BlockId), diff --git a/crates/nu-protocol/src/ast/expression.rs b/crates/nu-protocol/src/ast/expression.rs index 228bdf50ec..da79d9cc9d 100644 --- a/crates/nu-protocol/src/ast/expression.rs +++ b/crates/nu-protocol/src/ast/expression.rs @@ -6,6 +6,7 @@ pub struct Expression { pub expr: Expr, pub span: Span, pub ty: Type, + pub custom_completion: Option, } impl Expression { @@ -14,6 +15,7 @@ impl Expression { expr: Expr::Garbage, span, ty: Type::Unknown, + custom_completion: None, } } diff --git a/crates/nu-protocol/src/ast/import_pattern.rs b/crates/nu-protocol/src/ast/import_pattern.rs new file mode 100644 index 0000000000..1b7d6ad0fd --- /dev/null +++ b/crates/nu-protocol/src/ast/import_pattern.rs @@ -0,0 +1,14 @@ +use crate::Span; + +#[derive(Debug, Clone)] +pub enum ImportPatternMember { + Glob { span: Span }, + Name { name: Vec, span: Span }, + List { names: Vec<(Vec, Span)> }, +} + +#[derive(Debug, Clone)] +pub struct ImportPattern { + pub head: Vec, + pub members: Vec, +} diff --git a/crates/nu-protocol/src/ast/mod.rs b/crates/nu-protocol/src/ast/mod.rs index 4bd33ec765..67c9ce76e9 100644 --- a/crates/nu-protocol/src/ast/mod.rs +++ b/crates/nu-protocol/src/ast/mod.rs @@ -3,6 +3,7 @@ mod call; mod cell_path; mod expr; mod expression; +mod import_pattern; mod operator; mod pipeline; mod statement; @@ -12,6 +13,7 @@ pub use call::*; pub use cell_path::*; pub use expr::*; pub use expression::*; +pub use import_pattern::*; pub use operator::*; pub use pipeline::*; pub use statement::*; diff --git a/crates/nu-protocol/src/ast/operator.rs b/crates/nu-protocol/src/ast/operator.rs index c7c82eba41..690390888f 100644 --- a/crates/nu-protocol/src/ast/operator.rs +++ b/crates/nu-protocol/src/ast/operator.rs @@ -1,8 +1,9 @@ use crate::Span; +use serde::{Deserialize, Serialize}; use std::fmt::Display; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Operator { Equal, NotEqual, @@ -49,7 +50,7 @@ impl Display for Operator { } } -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum RangeInclusion { Inclusive, RightExclusive, @@ -59,6 +60,7 @@ pub enum RangeInclusion { pub struct RangeOperator { pub inclusion: RangeInclusion, pub span: Span, + pub next_op_span: Span, } impl Display for RangeOperator { diff --git a/crates/nu-protocol/src/engine/command.rs b/crates/nu-protocol/src/engine/command.rs index adac4ec916..ce322eb873 100644 --- a/crates/nu-protocol/src/engine/command.rs +++ b/crates/nu-protocol/src/engine/command.rs @@ -2,7 +2,7 @@ use crate::{ast::Call, value::Value, BlockId, Example, ShellError, Signature}; use super::EvaluationContext; -pub trait Command { +pub trait Command: Send + Sync { fn name(&self) -> &str; fn signature(&self) -> Signature { diff --git a/crates/nu-protocol/src/engine/engine_state.rs b/crates/nu-protocol/src/engine/engine_state.rs index 80d437a57c..f0920ed5c8 100644 --- a/crates/nu-protocol/src/engine/engine_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -1,7 +1,7 @@ use super::Command; -use crate::{ast::Block, BlockId, DeclId, Span, Type, VarId}; +use crate::{ast::Block, BlockId, DeclId, Signature, Span, Type, VarId}; use core::panic; -use std::{collections::HashMap, ops::Range, slice::Iter}; +use std::{collections::HashMap, slice::Iter}; pub struct EngineState { files: Vec<(String, usize, usize)>, @@ -17,6 +17,7 @@ pub struct ScopeFrame { vars: HashMap, VarId>, decls: HashMap, DeclId>, aliases: HashMap, Vec>, + modules: HashMap, BlockId>, } impl ScopeFrame { @@ -25,6 +26,7 @@ impl ScopeFrame { vars: HashMap::new(), decls: HashMap::new(), aliases: HashMap::new(), + modules: HashMap::new(), } } @@ -76,6 +78,9 @@ impl EngineState { for item in first.aliases.into_iter() { last.aliases.insert(item.0, item.1); } + for item in first.modules.into_iter() { + last.modules.insert(item.0, item.1); + } } } @@ -113,6 +118,11 @@ impl EngineState { } } + pub fn print_contents(&self) { + let string = String::from_utf8_lossy(&self.file_contents); + println!("{}", string); + } + pub fn find_decl(&self, name: &[u8]) -> Option { for scope in self.scope.iter().rev() { if let Some(decl_id) = scope.decls.get(name) { @@ -123,6 +133,24 @@ impl EngineState { None } + pub fn find_commands_by_prefix(&self, name: &[u8]) -> Vec> { + let mut output = vec![]; + + for scope in self.scope.iter().rev() { + for decl in &scope.decls { + if decl.0.starts_with(name) { + output.push(decl.0.clone()); + } + } + } + + output + } + + pub fn get_span_contents(&self, span: &Span) -> &[u8] { + &self.file_contents[span.start..span.end] + } + pub fn get_var(&self, var_id: VarId) -> &Type { self.vars .get(var_id) @@ -136,6 +164,21 @@ impl EngineState { .expect("internal error: missing declaration") } + pub fn get_decls(&self) -> Vec { + let mut output = vec![]; + for decl in self.decls.iter() { + if decl.get_block_id().is_none() { + let mut signature = (*decl).signature(); + signature.usage = decl.usage().to_string(); + signature.extra_usage = decl.extra_usage().to_string(); + + output.push(signature); + } + } + + output + } + pub fn get_block(&self, block_id: BlockId) -> &Block { self.blocks .get(block_id) @@ -272,6 +315,37 @@ impl<'a> StateWorkingSet<'a> { self.num_blocks() - 1 } + pub fn add_module(&mut self, name: &str, block: Block) -> BlockId { + let name = name.as_bytes().to_vec(); + + self.delta.blocks.push(block); + let block_id = self.num_blocks() - 1; + + let scope_frame = self + .delta + .scope + .last_mut() + .expect("internal error: missing required scope frame"); + + scope_frame.modules.insert(name, block_id); + + block_id + } + + pub fn activate_overlay(&mut self, overlay: Vec<(Vec, DeclId)>) { + // TODO: This will overwrite all existing definitions in a scope. When we add deactivate, + // we need to re-think how make it recoverable. + let scope_frame = self + .delta + .scope + .last_mut() + .expect("internal error: missing required scope frame"); + + for (name, decl_id) in overlay { + scope_frame.decls.insert(name, decl_id); + } + } + pub fn next_span_start(&self) -> usize { self.permanent_state.next_span_start() + self.delta.file_contents.len() } @@ -357,6 +431,22 @@ impl<'a> StateWorkingSet<'a> { None } + pub fn find_module(&self, name: &[u8]) -> Option { + for scope in self.delta.scope.iter().rev() { + if let Some(block_id) = scope.modules.get(name) { + return Some(*block_id); + } + } + + for scope in self.permanent_state.scope.iter().rev() { + if let Some(block_id) = scope.modules.get(name) { + return Some(*block_id); + } + } + + None + } + // pub fn update_decl(&mut self, decl_id: usize, block: Option) { // let decl = self.get_decl_mut(decl_id); // decl.body = block; @@ -496,6 +586,24 @@ impl<'a> StateWorkingSet<'a> { } } + pub fn find_commands_by_prefix(&self, name: &[u8]) -> Vec> { + let mut output = vec![]; + + for scope in self.delta.scope.iter().rev() { + for decl in &scope.decls { + if decl.0.starts_with(name) { + output.push(decl.0.clone()); + } + } + } + + let mut permanent = self.permanent_state.find_commands_by_prefix(name); + + output.append(&mut permanent); + + output + } + pub fn get_block(&self, block_id: BlockId) -> &Block { let num_permanent_blocks = self.permanent_state.num_blocks(); if block_id < num_permanent_blocks { @@ -513,95 +621,83 @@ impl<'a> StateWorkingSet<'a> { } } -impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> { - type FileId = usize; - - type Name = String; - - type Source = String; - - fn name(&'a self, id: Self::FileId) -> Result { - Ok(self.get_filename(id)) - } - - fn source( - &'a self, - id: Self::FileId, - ) -> Result { - Ok(self.get_file_source(id)) - } - - fn line_index( - &'a self, - id: Self::FileId, - byte_index: usize, - ) -> Result { - let source = self.get_file_source(id); - - let mut count = 0; - - for byte in source.bytes().enumerate() { - if byte.0 == byte_index { - // println!("count: {} for file: {} index: {}", count, id, byte_index); - return Ok(count); - } - if byte.1 == b'\n' { - count += 1; - } +impl<'a> miette::SourceCode for &StateWorkingSet<'a> { + fn read_span<'b>( + &'b self, + span: &miette::SourceSpan, + context_lines_before: usize, + context_lines_after: usize, + ) -> Result, miette::MietteError> { + let debugging = std::env::var("MIETTE_DEBUG").is_ok(); + if debugging { + let finding_span = "Finding span in StateWorkingSet"; + dbg!(finding_span, span); } - - // println!("count: {} for file: {} index: {}", count, id, byte_index); - Ok(count) - } - - fn line_range( - &'a self, - id: Self::FileId, - line_index: usize, - ) -> Result, codespan_reporting::files::Error> { - let source = self.get_file_source(id); - - let mut count = 0; - - let mut start = Some(0); - let mut end = None; - - for byte in source.bytes().enumerate() { - #[allow(clippy::comparison_chain)] - if count > line_index { - let start = start.expect("internal error: couldn't find line"); - let end = end.expect("internal error: couldn't find line"); - - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - return Ok(start..end); - } else if count == line_index { - end = Some(byte.0 + 1); + for (filename, start, end) in self.files() { + if debugging { + dbg!(&filename, start, end); } + if span.offset() >= *start && span.offset() + span.len() <= *end { + if debugging { + let found_file = "Found matching file"; + dbg!(found_file); + } + let our_span = Span { + start: *start, + end: *end, + }; + // We need to move to a local span because we're only reading + // the specific file contents via self.get_span_contents. + let local_span = (span.offset() - *start, span.len()).into(); + if debugging { + dbg!(&local_span); + } + let span_contents = self.get_span_contents(our_span); + if debugging { + dbg!(String::from_utf8_lossy(span_contents)); + } + let span_contents = span_contents.read_span( + &local_span, + context_lines_before, + context_lines_after, + )?; + let content_span = span_contents.span(); + // Back to "global" indexing + let retranslated = (content_span.offset() + start, content_span.len()).into(); + if debugging { + dbg!(&retranslated); + } - #[allow(clippy::comparison_chain)] - if byte.1 == b'\n' { - count += 1; - if count > line_index { - break; - } else if count == line_index { - start = Some(byte.0 + 1); + let data = span_contents.data(); + if filename == "" { + if debugging { + let success_cli = "Successfully read CLI span"; + dbg!(success_cli, String::from_utf8_lossy(data)); + } + return Ok(Box::new(miette::MietteSpanContents::new( + data, + retranslated, + span_contents.line(), + span_contents.column(), + span_contents.line_count(), + ))); + } else { + if debugging { + let success_file = "Successfully read file span"; + dbg!(success_file); + } + return Ok(Box::new(miette::MietteSpanContents::new_named( + filename.clone(), + data, + retranslated, + span_contents.line(), + span_contents.column(), + span_contents.line_count(), + ))); } } } - - match (start, end) { - (Some(start), Some(end)) => { - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - Ok(start..end) - } - _ => Err(codespan_reporting::files::Error::FileMissing), - } + Err(miette::MietteError::OutOfBounds) } } diff --git a/crates/nu-protocol/src/engine/evaluation_context.rs b/crates/nu-protocol/src/engine/evaluation_context.rs index 776a69fb12..9a0c1c6855 100644 --- a/crates/nu-protocol/src/engine/evaluation_context.rs +++ b/crates/nu-protocol/src/engine/evaluation_context.rs @@ -1,7 +1,7 @@ use super::EngineState; use std::{cell::RefCell, collections::HashMap, rc::Rc}; -use crate::{ShellError, Value, VarId}; +use crate::{ShellError, Signature, Value, VarId}; #[derive(Clone)] pub struct EvaluationContext { @@ -46,6 +46,10 @@ impl EvaluationContext { pub fn print_stack(&self) { self.stack.print_stack(); } + + pub fn get_commands_info(&self) -> Vec { + self.engine_state.borrow().get_decls() + } } #[derive(Debug)] @@ -104,6 +108,10 @@ impl Stack { }))) } + pub fn get_env_vars(&self) -> HashMap { + self.0.borrow().env_vars.clone() + } + pub fn print_stack(&self) { println!("===frame==="); println!("vars:"); diff --git a/crates/nu-protocol/src/shell_error.rs b/crates/nu-protocol/src/shell_error.rs index 678816b353..8307e99617 100644 --- a/crates/nu-protocol/src/shell_error.rs +++ b/crates/nu-protocol/src/shell_error.rs @@ -1,24 +1,81 @@ +use miette::Diagnostic; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + use crate::{ast::Operator, Span, Type}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Error, Diagnostic, Serialize, Deserialize)] pub enum ShellError { + #[error("Type mismatch during operation.")] + #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))] OperatorMismatch { + #[label = "type mismatch for operator"] op_span: Span, lhs_ty: Type, + #[label("{lhs_ty}")] lhs_span: Span, rhs_ty: Type, + #[label("{rhs_ty}")] rhs_span: Span, }, - UnsupportedOperator(Operator, Span), - UnknownOperator(String, Span), - ExternalNotSupported(Span), + + #[error("Unsupported operator: {0}.")] + #[diagnostic(code(nu::shell::unsupported_operator), url(docsrs))] + UnsupportedOperator(Operator, #[label = "unsupported operator"] Span), + + #[error("Unsupported operator: {0}.")] + #[diagnostic(code(nu::shell::unknown_operator), url(docsrs))] + UnknownOperator(String, #[label = "unsupported operator"] Span), + + #[error("External commands not yet supported")] + #[diagnostic(code(nu::shell::external_commands), url(docsrs))] + ExternalNotSupported(#[label = "external not supported"] Span), + + #[error("Internal error: {0}.")] + #[diagnostic(code(nu::shell::internal_error), url(docsrs))] InternalError(String), - VariableNotFoundAtRuntime(Span), - CantConvert(String, Span), - DivisionByZero(Span), - CannotCreateRange(Span), - AccessBeyondEnd(usize, Span), - AccessBeyondEndOfStream(Span), - IncompatiblePathAccess(String, Span), - CantFindColumn(Span), + + #[error("Variable not found")] + #[diagnostic(code(nu::shell::variable_not_found), url(docsrs))] + VariableNotFoundAtRuntime(#[label = "variable not found"] Span), + + #[error("Can't convert to {0}.")] + #[diagnostic(code(nu::shell::cant_convert), url(docsrs))] + CantConvert(String, #[label("can't convert to {0}")] Span), + + #[error("Division by zero.")] + #[diagnostic(code(nu::shell::division_by_zero), url(docsrs))] + DivisionByZero(#[label("division by zero")] Span), + + #[error("Can't convert range to countable values")] + #[diagnostic(code(nu::shell::range_to_countable), url(docsrs))] + CannotCreateRange(#[label = "can't convert to countable values"] Span), + + #[error("Row number too large (max: {0}).")] + #[diagnostic(code(nu::shell::access_beyond_end), url(docsrs))] + AccessBeyondEnd(usize, #[label = "too large"] Span), + + #[error("Row number too large.")] + #[diagnostic(code(nu::shell::access_beyond_end_of_stream), url(docsrs))] + AccessBeyondEndOfStream(#[label = "too large"] Span), + + #[error("Data cannot be accessed with a cell path")] + #[diagnostic(code(nu::shell::incompatible_path_access), url(docsrs))] + IncompatiblePathAccess(String, #[label("{0} doesn't support cell paths")] Span), + + #[error("Cannot find column")] + #[diagnostic(code(nu::shell::column_not_found), url(docsrs))] + CantFindColumn(#[label = "cannot find column"] Span), + + #[error("External command")] + #[diagnostic(code(nu::shell::external_command), url(docsrs))] + ExternalCommand(String, #[label("{0}")] Span), + + #[error("Unsupported input")] + #[diagnostic(code(nu::shell::unsupported_input), url(docsrs))] + UnsupportedInput(String, #[label("{0}")] Span), + + #[error("Flag not found")] + #[diagnostic(code(nu::shell::flag_not_found), url(docsrs))] + FlagNotFound(String, #[label("{0} not found")] Span), } diff --git a/crates/nu-protocol/src/span.rs b/crates/nu-protocol/src/span.rs index f1fd064355..1dd0b38982 100644 --- a/crates/nu-protocol/src/span.rs +++ b/crates/nu-protocol/src/span.rs @@ -1,9 +1,23 @@ -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +use miette::SourceSpan; +use serde::{Deserialize, Serialize}; + +pub struct Spanned { + pub item: T, + pub span: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Span { pub start: usize, pub end: usize, } +impl From for SourceSpan { + fn from(s: Span) -> Self { + Self::new(s.start.into(), (s.end - s.start).into()) + } +} + impl Span { pub fn new(start: usize, end: usize) -> Span { Span { start, end } diff --git a/crates/nu-protocol/src/syntax_shape.rs b/crates/nu-protocol/src/syntax_shape.rs index 3e3337cd0f..daed48c0c3 100644 --- a/crates/nu-protocol/src/syntax_shape.rs +++ b/crates/nu-protocol/src/syntax_shape.rs @@ -33,8 +33,11 @@ pub enum SyntaxShape { /// A glob pattern is allowed, eg `foo*` GlobPattern, + /// A module path pattern used for imports + ImportPattern, + /// A block is allowed, eg `{start this thing}` - Block, + Block(Option>), /// A table is allowed, eg `[[first, second]; [1, 2]]` Table, @@ -69,20 +72,25 @@ pub enum SyntaxShape { /// A general expression, eg `1 + 2` or `foo --bar` Expression, + + /// A custom shape with custom completion logic + Custom(Box, String), } impl SyntaxShape { pub fn to_type(&self) -> Type { match self { SyntaxShape::Any => Type::Unknown, - SyntaxShape::Block => Type::Block, + SyntaxShape::Block(_) => Type::Block, SyntaxShape::CellPath => Type::Unknown, + SyntaxShape::Custom(custom, _) => custom.to_type(), SyntaxShape::Duration => Type::Duration, SyntaxShape::Expression => Type::Unknown, SyntaxShape::FilePath => Type::FilePath, SyntaxShape::Filesize => Type::Filesize, SyntaxShape::FullCellPath => Type::Unknown, SyntaxShape::GlobPattern => Type::String, + SyntaxShape::ImportPattern => Type::Unknown, SyntaxShape::Int => Type::Int, SyntaxShape::List(x) => { let contents = x.to_type(); diff --git a/crates/nu-protocol/src/ty.rs b/crates/nu-protocol/src/ty.rs index bd441c3409..c11bafaca8 100644 --- a/crates/nu-protocol/src/ty.rs +++ b/crates/nu-protocol/src/ty.rs @@ -1,6 +1,8 @@ +use serde::{Deserialize, Serialize}; + use std::fmt::Display; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum Type { Int, Float, @@ -16,9 +18,11 @@ pub enum Type { Number, Nothing, Record(Vec, Vec), + Table, ValueStream, Unknown, Error, + Binary, } impl Display for Type { @@ -34,6 +38,7 @@ impl Display for Type { Type::Int => write!(f, "int"), Type::Range => write!(f, "range"), Type::Record(cols, vals) => write!(f, "record<{}, {:?}>", cols.join(", "), vals), + Type::Table => write!(f, "table"), Type::List(l) => write!(f, "list<{}>", l), Type::Nothing => write!(f, "nothing"), Type::Number => write!(f, "number"), @@ -41,6 +46,7 @@ impl Display for Type { Type::ValueStream => write!(f, "value stream"), Type::Unknown => write!(f, "unknown"), Type::Error => write!(f, "error"), + Type::Binary => write!(f, "binary"), } } } diff --git a/crates/nu-protocol/src/value/mod.rs b/crates/nu-protocol/src/value/mod.rs index 815a0fabfd..c6b61184b6 100644 --- a/crates/nu-protocol/src/value/mod.rs +++ b/crates/nu-protocol/src/value/mod.rs @@ -4,17 +4,18 @@ mod stream; pub use range::*; pub use row::*; +use serde::{Deserialize, Serialize}; pub use stream::*; use std::fmt::Debug; -use crate::ast::{PathMember, RangeInclusion}; +use crate::ast::PathMember; use crate::{span, BlockId, Span, Type}; use crate::ShellError; /// Core structured values that pass through the pipeline in engine-q -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum Value { Bool { val: bool, @@ -24,6 +25,14 @@ pub enum Value { val: i64, span: Span, }, + Filesize { + val: u64, + span: Span, + }, + Duration { + val: u64, + span: Span, + }, Range { val: Box, span: Span, @@ -59,6 +68,10 @@ pub enum Value { Error { error: ShellError, }, + Binary { + val: Vec, + span: Span, + }, } impl Value { @@ -76,6 +89,8 @@ impl Value { Value::Bool { span, .. } => *span, Value::Int { span, .. } => *span, Value::Float { span, .. } => *span, + Value::Filesize { span, .. } => *span, + Value::Duration { span, .. } => *span, Value::Range { span, .. } => *span, Value::String { span, .. } => *span, Value::Record { span, .. } => *span, @@ -83,6 +98,7 @@ impl Value { Value::Block { span, .. } => *span, Value::Stream { span, .. } => *span, Value::Nothing { span, .. } => *span, + Value::Binary { span, .. } => *span, } } @@ -92,6 +108,8 @@ impl Value { Value::Bool { span, .. } => *span = new_span, Value::Int { span, .. } => *span = new_span, Value::Float { span, .. } => *span = new_span, + Value::Filesize { span, .. } => *span = new_span, + Value::Duration { span, .. } => *span = new_span, Value::Range { span, .. } => *span = new_span, Value::String { span, .. } => *span = new_span, Value::Record { span, .. } => *span = new_span, @@ -100,6 +118,7 @@ impl Value { Value::Block { span, .. } => *span = new_span, Value::Nothing { span, .. } => *span = new_span, Value::Error { .. } => {} + Value::Binary { span, .. } => *span = new_span, } self @@ -111,6 +130,8 @@ impl Value { Value::Bool { .. } => Type::Bool, Value::Int { .. } => Type::Int, Value::Float { .. } => Type::Float, + Value::Filesize { .. } => Type::Filesize, + Value::Duration { .. } => Type::Duration, Value::Range { .. } => Type::Range, Value::String { .. } => Type::String, Value::Record { cols, vals, .. } => { @@ -121,6 +142,7 @@ impl Value { Value::Block { .. } => Type::Block, Value::Stream { .. } => Type::ValueStream, Value::Error { .. } => Type::Error, + Value::Binary { .. } => Type::Binary, } } @@ -130,21 +152,13 @@ impl Value { Value::Bool { val, .. } => val.to_string(), Value::Int { val, .. } => val.to_string(), Value::Float { val, .. } => val.to_string(), + Value::Filesize { val, .. } => format!("{} bytes", val), + Value::Duration { val, .. } => format!("{} ns", val), Value::Range { val, .. } => { - let vals: Vec = match (&val.from, &val.to) { - (Value::Int { val: from, .. }, Value::Int { val: to, .. }) => { - match val.inclusion { - RangeInclusion::Inclusive => (*from..=*to).collect(), - RangeInclusion::RightExclusive => (*from..*to).collect(), - } - } - _ => Vec::new(), - }; - format!( "range: [{}]", - vals.iter() - .map(|x| x.to_string()) + val.into_iter() + .map(|x| x.into_string()) .collect::>() .join(", ") ) @@ -169,6 +183,38 @@ impl Value { Value::Block { val, .. } => format!("", val), Value::Nothing { .. } => String::new(), Value::Error { error } => format!("{:?}", error), + Value::Binary { val, .. } => format!("{:?}", val), + } + } + + pub fn collect_string(self) -> String { + match self { + Value::Bool { val, .. } => val.to_string(), + Value::Int { val, .. } => val.to_string(), + Value::Float { val, .. } => val.to_string(), + Value::Filesize { val, .. } => format!("{} bytes", val), + Value::Duration { val, .. } => format!("{} ns", val), + Value::Range { val, .. } => val + .into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", "), + Value::String { val, .. } => val, + Value::Stream { stream, .. } => stream.collect_string(), + Value::List { vals: val, .. } => val + .into_iter() + .map(|x| x.collect_string()) + .collect::>() + .join("\n"), + Value::Record { vals, .. } => vals + .into_iter() + .map(|y| y.collect_string()) + .collect::>() + .join("\n"), + Value::Block { val, .. } => format!("", val), + Value::Nothing { .. } => String::new(), + Value::Error { error } => format!("{:?}", error), + Value::Binary { val, .. } => format!("{:?}", val), } } @@ -180,9 +226,9 @@ impl Value { } /// Follow a given column path into the value: for example accessing nth elements in a stream or list - pub fn follow_cell_path(self, column_path: &[PathMember]) -> Result { + pub fn follow_cell_path(self, cell_path: &[PathMember]) -> Result { let mut current = self; - for member in column_path { + for member in cell_path { // FIXME: this uses a few extra clones for simplicity, but there may be a way // to traverse the path without them match member { @@ -278,6 +324,24 @@ impl Value { Ok(current) } + + pub fn string(s: &str, span: Span) -> Value { + Value::String { + val: s.into(), + span, + } + } + + pub fn is_true(&self) -> bool { + matches!(self, Value::Bool { val: true, .. }) + } + + pub fn columns(&self) -> Vec { + match self { + Value::Record { cols, .. } => cols.clone(), + _ => vec![], + } + } } impl PartialEq for Value { diff --git a/crates/nu-protocol/src/value/range.rs b/crates/nu-protocol/src/value/range.rs index 6c7de36166..369ff03e87 100644 --- a/crates/nu-protocol/src/value/range.rs +++ b/crates/nu-protocol/src/value/range.rs @@ -1,13 +1,110 @@ -use crate::{ast::RangeInclusion, *}; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; /// A Range is an iterator over integers. -#[derive(Debug, Clone, PartialEq)] +use crate::{ + ast::{RangeInclusion, RangeOperator}, + *, +}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Range { pub from: Value, + pub incr: Value, pub to: Value, pub inclusion: RangeInclusion, } +impl Range { + pub fn new( + expr_span: Span, + from: Value, + next: Value, + to: Value, + operator: &RangeOperator, + ) -> Result { + // Select from & to values if they're not specified + // TODO: Replace the placeholder values with proper min/max based on data type + let from = if let Value::Nothing { .. } = from { + Value::Int { + val: 0i64, + span: Span::unknown(), + } + } else { + from + }; + + let to = if let Value::Nothing { .. } = to { + if let Ok(Value::Bool { val: true, .. }) = next.lt(expr_span, &from) { + Value::Int { + val: -100i64, + span: Span::unknown(), + } + } else { + Value::Int { + val: 100i64, + span: Span::unknown(), + } + } + } else { + to + }; + + // Check if the range counts up or down + let moves_up = matches!(from.lte(expr_span, &to), Ok(Value::Bool { val: true, .. })); + + // Convert the next value into the inctement + let incr = if let Value::Nothing { .. } = next { + if moves_up { + Value::Int { + val: 1i64, + span: Span::unknown(), + } + } else { + Value::Int { + val: -1i64, + span: Span::unknown(), + } + } + } else { + next.sub(operator.next_op_span, &from)? + }; + + let zero = Value::Int { + val: 0i64, + span: Span::unknown(), + }; + + // Increment must be non-zero, otherwise we iterate forever + if matches!(incr.eq(expr_span, &zero), Ok(Value::Bool { val: true, .. })) { + return Err(ShellError::CannotCreateRange(expr_span)); + } + + // If to > from, then incr > 0, otherwise we iterate forever + if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = ( + to.gt(operator.span, &from)?, + incr.gt(operator.next_op_span, &zero)?, + ) { + return Err(ShellError::CannotCreateRange(expr_span)); + } + + // If to < from, then incr < 0, otherwise we iterate forever + if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = ( + to.lt(operator.span, &from)?, + incr.lt(operator.next_op_span, &zero)?, + ) { + return Err(ShellError::CannotCreateRange(expr_span)); + } + + Ok(Range { + from, + incr, + to, + inclusion: operator.inclusion, + }) + } +} + impl IntoIterator for Range { type Item = Value; @@ -26,8 +123,7 @@ pub struct RangeIterator { span: Span, is_end_inclusive: bool, moves_up: bool, - one: Value, - negative_one: Value, + incr: Value, done: bool, } @@ -53,41 +149,66 @@ impl RangeIterator { span, is_end_inclusive: matches!(range.inclusion, RangeInclusion::Inclusive), done: false, - one: Value::Int { val: 1, span }, - negative_one: Value::Int { val: -1, span }, + incr: range.incr, } } } +// Compare two floating point numbers. The decision interval for equality is dynamically scaled +// as the value being compared increases in magnitude. +fn compare_floats(val: f64, other: f64) -> Option { + let prec = f64::EPSILON.max(val.abs() * f64::EPSILON); + + if (other - val).abs() < prec { + return Some(Ordering::Equal); + } + + val.partial_cmp(&other) +} + impl Iterator for RangeIterator { type Item = Value; fn next(&mut self) -> Option { - use std::cmp::Ordering; if self.done { return None; } let ordering = if matches!(self.end, Value::Nothing { .. }) { - Ordering::Less + Some(Ordering::Less) } else { match (&self.curr, &self.end) { - (Value::Int { val: x, .. }, Value::Int { val: y, .. }) => x.cmp(y), - // (Value::Float { val: x, .. }, Value::Float { val: y, .. }) => x.cmp(y), - // (Value::Float { val: x, .. }, Value::Int { val: y, .. }) => x.cmp(y), - // (Value::Int { val: x, .. }, Value::Float { val: y, .. }) => x.cmp(y), - _ => { - self.done = true; - return Some(Value::Error { - error: ShellError::CannotCreateRange(self.span), - }); + (Value::Int { val: curr, .. }, Value::Int { val: end, .. }) => Some(curr.cmp(end)), + (Value::Float { val: curr, .. }, Value::Float { val: end, .. }) => { + compare_floats(*curr, *end) } + (Value::Float { val: curr, .. }, Value::Int { val: end, .. }) => { + compare_floats(*curr, *end as f64) + } + (Value::Int { val: curr, .. }, Value::Float { val: end, .. }) => { + compare_floats(*curr as f64, *end) + } + _ => None, } }; - if self.moves_up - && (ordering == Ordering::Less || self.is_end_inclusive && ordering == Ordering::Equal) + let ordering = if let Some(ord) = ordering { + ord + } else { + self.done = true; + return Some(Value::Error { + error: ShellError::CannotCreateRange(self.span), + }); + }; + + let desired_ordering = if self.moves_up { + Ordering::Less + } else { + Ordering::Greater + }; + + if (ordering == desired_ordering) || (self.is_end_inclusive && ordering == Ordering::Equal) { - let next_value = self.curr.add(self.span, &self.one); + let next_value = self.curr.add(self.span, &self.incr); let mut next = match next_value { Ok(result) => result, @@ -99,22 +220,6 @@ impl Iterator for RangeIterator { }; std::mem::swap(&mut self.curr, &mut next); - Some(next) - } else if !self.moves_up - && (ordering == Ordering::Greater - || self.is_end_inclusive && ordering == Ordering::Equal) - { - let next_value = self.curr.add(self.span, &self.negative_one); - - let mut next = match next_value { - Ok(result) => result, - Err(error) => { - self.done = true; - return Some(Value::Error { error }); - } - }; - std::mem::swap(&mut self.curr, &mut next); - Some(next) } else { None diff --git a/crates/nu-protocol/src/value/stream.rs b/crates/nu-protocol/src/value/stream.rs index ad5a27208f..4ac557247f 100644 --- a/crates/nu-protocol/src/value/stream.rs +++ b/crates/nu-protocol/src/value/stream.rs @@ -1,6 +1,8 @@ use crate::*; use std::{cell::RefCell, fmt::Debug, rc::Rc}; +use serde::{Deserialize, Serialize}; + #[derive(Clone)] pub struct ValueStream(pub Rc>>); @@ -14,6 +16,12 @@ impl ValueStream { ) } + pub fn collect_string(self) -> String { + self.map(|x: Value| x.collect_string()) + .collect::>() + .join("\n") + } + pub fn from_stream(input: impl Iterator + 'static) -> ValueStream { ValueStream(Rc::new(RefCell::new(input))) } @@ -30,12 +38,31 @@ impl Iterator for ValueStream { fn next(&mut self) -> Option { { - let mut iter = self.0.borrow_mut(); - iter.next() + self.0.borrow_mut().next() } } } +impl Serialize for ValueStream { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + // FIXME: implement these + todo!() + } +} + +impl<'de> Deserialize<'de> for ValueStream { + fn deserialize(_deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + // FIXME: implement these + todo!() + } +} + pub trait IntoValueStream { fn into_value_stream(self) -> ValueStream; } diff --git a/crates/nu-table/.gitignore b/crates/nu-table/.gitignore new file mode 100644 index 0000000000..4c234e523b --- /dev/null +++ b/crates/nu-table/.gitignore @@ -0,0 +1,22 @@ +/target +/scratch +**/*.rs.bk +history.txt +tests/fixtures/nuplayground +crates/*/target + +# Debian/Ubuntu +debian/.debhelper/ +debian/debhelper-build-stamp +debian/files +debian/nu.substvars +debian/nu/ + +# macOS junk +.DS_Store + +# JetBrains' IDE items +.idea/* + +# VSCode's IDE items +.vscode/* diff --git a/crates/nu-table/Cargo.toml b/crates/nu-table/Cargo.toml new file mode 100644 index 0000000000..e831ec3f16 --- /dev/null +++ b/crates/nu-table/Cargo.toml @@ -0,0 +1,18 @@ +[package] +authors = ["The Nu Project Contributors"] +description = "Nushell table printing" +edition = "2018" +license = "MIT" +name = "nu-table" +version = "0.36.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "table" +path = "src/main.rs" + +[dependencies] +nu-ansi-term = "0.36.0" + +regex = "1.4" +unicode-width = "0.1.8" diff --git a/crates/nu-table/src/lib.rs b/crates/nu-table/src/lib.rs new file mode 100644 index 0000000000..661d7ddde7 --- /dev/null +++ b/crates/nu-table/src/lib.rs @@ -0,0 +1,5 @@ +mod table; +mod wrap; + +pub use table::{draw_table, StyledString, Table, TextStyle, Theme}; +pub use wrap::Alignment; diff --git a/crates/nu-table/src/main.rs b/crates/nu-table/src/main.rs new file mode 100644 index 0000000000..638582a1fa --- /dev/null +++ b/crates/nu-table/src/main.rs @@ -0,0 +1,86 @@ +use nu_table::{draw_table, StyledString, Table, TextStyle, Theme}; +use std::collections::HashMap; + +fn main() { + let args: Vec<_> = std::env::args().collect(); + let mut width = 0; + + if args.len() > 1 { + // Width in terminal characters + width = args[1].parse::().expect("Need a width in columns"); + } + + if width < 4 { + println!("Width must be greater than or equal to 4, setting width to 80"); + width = 80; + } + + // The mocked up table data + let (table_headers, row_data) = make_table_data(); + // The table headers + let headers = vec_of_str_to_vec_of_styledstr(&table_headers, true); + // The table rows + let rows = vec_of_str_to_vec_of_styledstr(&row_data, false); + // The table itself + let table = Table::new(headers, vec![rows; 3], Theme::rounded()); + // FIXME: Config isn't available from here so just put these here to compile + let color_hm: HashMap = HashMap::new(); + // Capture the table as a string + let output_table = draw_table(&table, width, &color_hm); + // Draw the table + println!("{}", output_table) +} + +fn make_table_data() -> (Vec<&'static str>, Vec<&'static str>) { + let table_headers = vec![ + "category", + "description", + "emoji", + "ios_version", + "unicode_version", + "aliases", + "tags", + "category2", + "description2", + "emoji2", + "ios_version2", + "unicode_version2", + "aliases2", + "tags2", + ]; + + let row_data = vec![ + "Smileys & Emotion", + "grinning face", + "😀", + "6", + "6.1", + "grinning", + "smile", + "Smileys & Emotion", + "grinning face", + "😀", + "6", + "6.1", + "grinning", + "smile", + ]; + + (table_headers, row_data) +} + +fn vec_of_str_to_vec_of_styledstr(data: &[&str], is_header: bool) -> Vec { + let mut v = vec![]; + + for x in data { + if is_header { + v.push(StyledString::new( + String::from(*x), + TextStyle::default_header(), + )) + } else { + v.push(StyledString::new(String::from(*x), TextStyle::basic_left())) + } + } + v +} diff --git a/crates/nu-table/src/table.rs b/crates/nu-table/src/table.rs new file mode 100644 index 0000000000..3110e2751d --- /dev/null +++ b/crates/nu-table/src/table.rs @@ -0,0 +1,1260 @@ +use crate::wrap::{column_width, split_sublines, wrap, Alignment, Subline, WrappedCell}; +use nu_ansi_term::{Color, Style}; +use std::collections::HashMap; +use std::fmt::Write; + +enum SeparatorPosition { + Top, + Middle, + Bottom, +} + +#[derive(Debug)] +pub struct Table { + pub headers: Vec, + pub data: Vec>, + pub theme: Theme, +} + +#[derive(Debug, Clone)] +pub struct StyledString { + pub contents: String, + pub style: TextStyle, +} + +impl StyledString { + pub fn new(contents: String, style: TextStyle) -> StyledString { + StyledString { contents, style } + } + + pub fn set_style(&mut self, style: TextStyle) { + self.style = style; + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TextStyle { + pub alignment: Alignment, + pub color_style: Option