mirror of
https://github.com/nushell/nushell
synced 2024-12-30 23:08:56 +00:00
7ac3e97bfe
# Description Currently, the `into sqlite` command collects the entire input stream into a single Value, which soaks up the entire input into memory, before it ever tries to write anything to the DB. This is very problematic for large inputs; for example, I tried transforming a multi-gigabyte CSV file into SQLite, and before I knew what was happening, my system's memory was completely exhausted, and I had to hard reboot to recover. This PR fixes this problem by working directly with the pipeline stream, inserting into the DB as values are read from the stream. In order to facilitate working with the stream directly, I introduced a new `Table` struct to store the connection and a few configuration parameters, as well as to make it easier to lazily create the table on the first read value. In addition to the purely functional fixes, a few other changes were made to the serialization and user facing behavior. ### Serialization Much of the preexisting code was focused on generating the exact text needed for a SQL statement. This is unneeded and less safe than using the `rusqlite` crate's serialization for native Rust types along with prepared statements. ### User-Facing Changes Currently, the command is very liberal in the input types it accepts. The strategy is basically if it is a record, try to follow its structure and make an analogous SQL row, which is pretty reasonable. However, when it's not a record, it basically tries to guess what the user wanted and just makes a single column table and serializes the value into that one column, whatever type it may be. This has been changed so that it only accepts records as input. If the user wants to serialize non-record types into SQL, then they must explicitly opt into doing this by constructing a record or table with it first. For a utility for inserting data into SQL, I think it makes more sense to let the user choose how to convert their data, rather than make a choice for them that may surprise them. However, I understand this may be a controversial change. If the maintainers don't agree, I can change this back. #### Long switch names The `file_name` and `table_name` long form switches are currently snake_case and expect to be as such at the command line. These have been changed to kebab-case to be more conventional. # Tests + Formatting To test the memory consumption, I used [this publicly available index of all Wikipedia articles](https://dumps.wikimedia.org/enwiki/20230820/), using the first 10,000, 100,000, and 1,000,000 entries, in that order. I ran the following script to benchmark the changes against the current stable release: ```nu #!/usr/bin/nu # let shellbin = $"($env.HOME)/src/nushell/target/aarch64-linux-android/release/nu" let shellbin = "nu" const dbpath = 'enwiki-index.db' [10000, 100000, 1000000] | each {|rows| rm -f $dbpath; do { time -f '%M %e %U %S' $shellbin -c ( $"bzip2 -cdk ~/enwiki-20230820-pages-articles-multistream-index.txt.bz2 | head -n ($rows) | lines | parse '{offset}:{id}:{title}' | update cells -c [offset, id] { into int } | into sqlite ($dbpath)" ) } | complete | get stderr | str trim | parse '{rss_max} {real} {user} {kernel}' | update cells -c [rss_max] { $"($in)kb" | into filesize } | update cells -c [real, user, kernel] { $"($in)sec" | into duration } | insert rows $rows | roll right } | flatten | to nuon ``` This yields the following results Current stable release: |rows|rss_max|real|user|kernel| |-|-|-|-|-| |10000|53.6 MiB|770ms|460ms|420ms| |100000|209.6 MiB|6sec 940ms|3sec 740ms|4sec 380ms| |1000000|1.7 GiB|1min 8sec 810ms|38sec 690ms|42sec 550ms| This PR: |rows|rss_max|real|user|kernel| |-|-|-|-|-| |10000|38.2 MiB|780ms|440ms|410ms| |100000|39.8 MiB|6sec 450ms|3sec 530ms|4sec 160ms| |1000000|39.8 MiB|1min 3sec 230ms|37sec 440ms|40sec 180ms| # Note I started this branch kind of at the same time as my others, but I understand the feedback that smaller PRs are preferred. Let me know if it would be better to split this up. I do think the scope of the changes are on the bigger side even without the behavior changes I mentioned, so I'm not sure if that will help this particular PR very much, but I'm happy to oblige on request.
126 lines
1.6 KiB
Rust
126 lines
1.6 KiB
Rust
mod alias;
|
|
mod all;
|
|
mod any;
|
|
mod append;
|
|
mod assignment;
|
|
mod break_;
|
|
mod cal;
|
|
mod cd;
|
|
mod compact;
|
|
mod complete;
|
|
mod config_env_default;
|
|
mod config_nu_default;
|
|
mod continue_;
|
|
mod conversions;
|
|
mod cp;
|
|
mod database;
|
|
mod date;
|
|
mod debug_info;
|
|
mod def;
|
|
mod default;
|
|
mod detect_columns;
|
|
mod do_;
|
|
mod drop;
|
|
mod each;
|
|
mod echo;
|
|
mod empty;
|
|
mod error_make;
|
|
mod every;
|
|
mod exec;
|
|
mod export_def;
|
|
mod fill;
|
|
mod find;
|
|
mod first;
|
|
mod flatten;
|
|
mod for_;
|
|
#[cfg(feature = "extra")]
|
|
mod format;
|
|
mod generate;
|
|
mod get;
|
|
mod glob;
|
|
mod group_by;
|
|
mod hash_;
|
|
mod headers;
|
|
mod help;
|
|
mod histogram;
|
|
mod insert;
|
|
mod inspect;
|
|
mod into_datetime;
|
|
mod into_filesize;
|
|
mod into_int;
|
|
mod join;
|
|
mod last;
|
|
mod length;
|
|
mod let_;
|
|
mod lines;
|
|
mod loop_;
|
|
mod ls;
|
|
mod match_;
|
|
mod math;
|
|
mod merge;
|
|
mod mkdir;
|
|
mod mktemp;
|
|
mod move_;
|
|
mod mut_;
|
|
mod network;
|
|
mod nu_check;
|
|
mod open;
|
|
mod par_each;
|
|
mod parse;
|
|
mod path;
|
|
mod platform;
|
|
mod prepend;
|
|
mod print;
|
|
#[cfg(feature = "sqlite")]
|
|
mod query;
|
|
mod random;
|
|
mod range;
|
|
mod redirection;
|
|
mod reduce;
|
|
mod reject;
|
|
mod rename;
|
|
mod return_;
|
|
mod reverse;
|
|
mod rm;
|
|
#[cfg(feature = "extra")]
|
|
mod roll;
|
|
#[cfg(feature = "extra")]
|
|
mod rotate;
|
|
mod run_external;
|
|
mod save;
|
|
mod select;
|
|
mod semicolon;
|
|
mod seq;
|
|
mod seq_char;
|
|
mod skip;
|
|
mod sort;
|
|
mod sort_by;
|
|
mod source_env;
|
|
mod split_by;
|
|
mod split_column;
|
|
mod split_row;
|
|
mod str_;
|
|
mod table;
|
|
mod take;
|
|
mod terminal;
|
|
mod to_text;
|
|
mod touch;
|
|
mod transpose;
|
|
mod try_;
|
|
mod ucp;
|
|
#[cfg(unix)]
|
|
mod ulimit;
|
|
mod umkdir;
|
|
mod uniq;
|
|
mod uniq_by;
|
|
mod update;
|
|
mod upsert;
|
|
mod url;
|
|
mod use_;
|
|
mod where_;
|
|
#[cfg(feature = "which-support")]
|
|
mod which;
|
|
mod while_;
|
|
mod with_env;
|
|
mod wrap;
|
|
mod zip;
|