Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2024-11-10 07:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

Remove dataframes crate and feature (#12889)

Browse source 
# Description
Removes the old `nu-cmd-dataframe` crate in favor of the polars plugin.
As such, this PR also removes the `dataframe` feature, related CI, and
full releases of nushell.
This commit is contained in:
Ian Manske 2024-05-20 17:22:08 +00:00 committed by GitHub
parent 4f69ba172e
commit 905e3d0715
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
156 changed files with 56 additions and 19464 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
.github/workflows/ci.yml vendored
View file

@ -29,17 +29,6 @@ jobs:
# instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are
# removed and we're only building the `polars` plugin instead
platform: [windows-latest, macos-13, ubuntu-20.04]
feature: [default, dataframe]
include:
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-13
feature: dataframe
runs-on: ${{ matrix.platform }}
@ -48,43 +37,31 @@ jobs:
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: cargo fmt
run: cargo fmt --all -- --check
# If changing these settings also change toolkit.nu
- name: Clippy
run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS
# In tests we don't have to deny unwrap
- name: Clippy of tests
run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings
- name: Clippy of benchmarks
run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings
tests:
strategy:
fail-fast: true
matrix:
platform: [windows-latest, macos-latest, ubuntu-20.04]
feature: [default, dataframe]
include:
# linux CI cannot handle clipboard feature
- default-flags: ""
# linux CI cannot handle clipboard feature
- platform: ubuntu-20.04
default-flags: "--no-default-features --features=default-no-clipboard"
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-latest
feature: dataframe
runs-on: ${{ matrix.platform }}
@ -93,12 +70,9 @@ jobs:
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Tests
run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }}
- name: Check for clean repo
shell: bash
run: |
@ -125,8 +99,6 @@ jobs:
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Install Nushell
run: cargo install --path . --locked --no-default-features
@ -178,8 +150,6 @@ jobs:
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Clippy
run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS

129
.github/workflows/nightly-build.yml vendored
View file

@ -84,41 +84,30 @@ jobs:
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: ''
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-20.04
target_rustflags: ''
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
runs-on: ${{matrix.os}}
@ -147,12 +136,10 @@ jobs:
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
@ -184,122 +171,6 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
needs: prepare
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.5
with:
ref: main
fetch-depth: 0
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.93.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
uses: JasonEtco/create-an-issue@v2.9.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
update_existing: true
search_existing: open
filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
- name: Set Outputs of Short SHA
id: vars
run: |
echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
sha_short=$(git rev-parse --short HEAD)
echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
# REF: https://github.com/marketplace/actions/gh-release
# Create a release only in nushell/nightly repo
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.5
if: ${{ startsWith(github.repository, 'nushell/nightly') }}
with:
draft: false
prerelease: true
name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
tag_name: nightly-${{ steps.vars.outputs.sha_short }}
body: |
This is a NIGHTLY build of Nushell.
It is NOT recommended for production use.
files: ${{ steps.nu.outputs.archive }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cleanup:
name: Cleanup
# Should only run in nushell/nightly repo

69
.github/workflows/release-pkg.nu vendored
View file

@ -9,7 +9,6 @@
# Instructions for manually creating an MSI for Winget Releases when they fail
# Added 2022-11-29 when Windows packaging wouldn't work
# Updated again on 2023-02-23 because msis are still failing validation
# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
# To run this manual for windows here are the steps I take
# checkout the release you want to publish
# 1. git checkout 0.86.0
@ -17,28 +16,26 @@
# 2. $env:CARGO_TARGET_DIR = ""
# 2. hide-env CARGO_TARGET_DIR
# 3. $env.TARGET = 'x86_64-pc-windows-msvc'
# 4. $env.TARGET_RUSTFLAGS = ''
# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 7. $env.OS = 'windows-latest'
# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
# 4. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 6. $env.OS = 'windows-latest'
# make sure 7z.exe is in your path https://www.7-zip.org/download.html
# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# make sure aria2c.exe is in your path https://github.com/aria2/aria2
# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# make sure you have the wixtools installed https://wixtoolset.org/
# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
# folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
# 12. $env._EXTRA_ = 'bin'
# 13. source .github\workflows\release-pkg.nu
# 14. cd ..
# 15. $env._EXTRA_ = 'msi'
# 16. source .github\workflows\release-pkg.nu
# 10. $env._EXTRA_ = 'bin'
# 11. source .github\workflows\release-pkg.nu
# 12. cd ..
# 13. $env._EXTRA_ = 'msi'
# 14. source .github\workflows\release-pkg.nu
# After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
# by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
# on the winget-pkgs PR. To generate the hash, run this command
# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
@ -48,31 +45,15 @@ let os = $env.OS
let target = $env.TARGET
# Repo source dir like `/home/runner/work/nushell/nushell`
let src = $env.GITHUB_WORKSPACE
let flags = $env.TARGET_RUSTFLAGS
let dist = $'($env.GITHUB_WORKSPACE)/output'
let version = (open Cargo.toml | get package.version)
print $'Debugging info:'
print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
const FULL_RLS_NAMING = {
x86_64-apple-darwin: 'x86_64-darwin-full',
aarch64-apple-darwin: 'aarch64-darwin-full',
x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
}
print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b
# $env
let USE_UBUNTU = $os starts-with ubuntu
let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
@ -91,23 +72,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
'aarch64-unknown-linux-gnu' => {
sudo apt-get install gcc-aarch64-linux-gnu -y
$env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
cargo-build-nu $flags
cargo-build-nu
}
'riscv64gc-unknown-linux-gnu' => {
sudo apt-get install gcc-riscv64-linux-gnu -y
$env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
cargo-build-nu $flags
cargo-build-nu
}
'armv7-unknown-linux-gnueabihf' => {
sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
$env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
cargo-build-nu $flags
cargo-build-nu
}
_ => {
# musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
# Actually just for x86_64-unknown-linux-musl target
if $USE_UBUNTU { sudo apt install musl-tools -y }
cargo-build-nu $flags
cargo-build-nu
}
}
}
@ -116,7 +97,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
# Build for Windows without static-link-openssl feature
# ----------------------------------------------------------------------------
if $os in ['windows-latest'] {
cargo-build-nu $flags
cargo-build-nu
}
# ----------------------------------------------------------------------------
@ -162,7 +143,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line
if $os in ['macos-latest'] or $USE_UBUNTU {
let files = (ls | get name)
let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
let dest = $'($bin)-($version)-($target)'
let archive = $'($dist)/($dest).tar.gz'
mkdir $dest
@ -177,7 +158,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
} else if $os == 'windows-latest' {
let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
let releaseStem = $'($bin)-($version)-($target)'
print $'(char nl)Download less related stuffs...'; hr-line
aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
@ -214,20 +195,12 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
}
}
def 'cargo-build-nu' [ options: string ] {
if ($options | str trim | is-empty) {
def 'cargo-build-nu' [] {
if $os == 'windows-latest' {
cargo build --release --all --target $target
} else {
cargo build --release --all --target $target --features=static-link-openssl
}
} else {
if $os == 'windows-latest' {
cargo build --release --all --target $target $options
} else {
cargo build --release --all --target $target --features=static-link-openssl $options
}
}
}
# Print a horizontal line marker

103
.github/workflows/release.yml vendored
View file

@ -34,41 +34,30 @@ jobs:
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: ''
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-20.04
target_rustflags: ''
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
runs-on: ${{matrix.os}}
@ -95,102 +84,10 @@ jobs:
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.5
if: ${{ startsWith(github.ref, 'refs/tags/') }}
with:
draft: true
files: ${{ steps.nu.outputs.archive }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.5
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
cache: false
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.93.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
- name: Publish Archive

10
CONTRIBUTING.md
View file

@ -71,11 +71,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo run
```
- Build and run with dataframe support.
```nushell
cargo run --features=dataframe
```
- Run Clippy on Nushell:
```nushell
@ -93,11 +88,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo test --workspace
```
along with dataframe tests
```nushell
cargo test --workspace --features=dataframe
```
or via the `toolkit.nu` command:
```nushell
use toolkit.nu test

24
Cargo.lock generated
View file

@ -2782,7 +2782,6 @@ dependencies = [
"nix",
"nu-cli",
"nu-cmd-base",
"nu-cmd-dataframe",
"nu-cmd-extra",
"nu-cmd-lang",
"nu-cmd-plugin",
@ -2870,29 +2869,6 @@ dependencies = [
"nu-protocol",
]
[[package]]
name = "nu-cmd-dataframe"
version = "0.93.1"
dependencies = [
"chrono",
"chrono-tz 0.8.6",
"fancy-regex",
"indexmap",
"nu-cmd-lang",
"nu-engine",
"nu-parser",
"nu-protocol",
"num",
"polars",
"polars-arrow",
"polars-io",
"polars-ops",
"polars-plan",
"polars-utils",
"serde",
"sqlparser 0.45.0",
]
[[package]]
name = "nu-cmd-extra"
version = "0.93.1"

7
Cargo.toml
View file

@ -32,7 +32,6 @@ members = [
"crates/nu-cmd-extra",
"crates/nu-cmd-lang",
"crates/nu-cmd-plugin",
"crates/nu-cmd-dataframe",
"crates/nu-command",
"crates/nu-color-config",
"crates/nu-explore",
@ -179,9 +178,6 @@ nu-cli = { path = "./crates/nu-cli", version = "0.93.1" }
nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.93.1" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.93.1" }
nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.93.1", optional = true }
nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.93.1", features = [
"dataframe",
], optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.93.1" }
nu-command = { path = "./crates/nu-command", version = "0.93.1" }
nu-engine = { path = "./crates/nu-engine", version = "0.93.1" }
@ -271,9 +267,6 @@ system-clipboard = [
which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
# Dataframe feature for nushell
dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
# SQLite commands for nushell
sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]

75
crates/nu-cmd-dataframe/Cargo.toml
View file

@ -1,75 +0,0 @@
[package]
authors = ["The Nushell Project Developers"]
description = "Nushell's dataframe commands based on polars."
edition = "2021"
license = "MIT"
name = "nu-cmd-dataframe"
repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
version = "0.93.1"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
bench = false
[dependencies]
nu-engine = { path = "../nu-engine", version = "0.93.1" }
nu-parser = { path = "../nu-parser", version = "0.93.1" }
nu-protocol = { path = "../nu-protocol", version = "0.93.1" }
# Potential dependencies for extras
chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
chrono-tz = { workspace = true }
fancy-regex = { workspace = true }
indexmap = { workspace = true }
num = { version = "0.4", optional = true }
serde = { workspace = true, features = ["derive"] }
# keep sqlparser at 0.39.0 until we can update polars
sqlparser = { version = "0.45", optional = true }
polars-io = { version = "0.39", features = ["avro"], optional = true }
polars-arrow = { version = "0.39", optional = true }
polars-ops = { version = "0.39", optional = true }
polars-plan = { version = "0.39", features = ["regex"], optional = true }
polars-utils = { version = "0.39", optional = true }
[dependencies.polars]
features = [
"arg_where",
"checked_arithmetic",
"concat_str",
"cross_join",
"csv",
"cum_agg",
"dtype-categorical",
"dtype-datetime",
"dtype-struct",
"dtype-i8",
"dtype-i16",
"dtype-u8",
"dtype-u16",
"dynamic_group_by",
"ipc",
"is_in",
"json",
"lazy",
"object",
"parquet",
"random",
"rolling_window",
"rows",
"serde",
"serde-lazy",
"strings",
"temporal",
"to_dummies",
]
default-features = false
optional = true
version = "0.39"
[features]
dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
default = []
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" }

21
crates/nu-cmd-dataframe/LICENSE
View file

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2019 - 2023 The Nushell Project Developers
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

12
crates/nu-cmd-dataframe/src/dataframe/README.md
View file

@ -1,12 +0,0 @@
# Dataframe
This dataframe directory holds all of the definitions of the dataframe data structures and commands.
There are three sections of commands:
* [eager](./eager)
* [series](./series)
* [values](./values)
For more details see the
[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html)

134
crates/nu-cmd-dataframe/src/dataframe/eager/append.rs
View file

@ -1,134 +0,0 @@
use crate::dataframe::values::{Axis, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct AppendDF;
impl Command for AppendDF {
fn name(&self) -> &str {
"dfr append"
}
fn usage(&self) -> &str {
"Appends a new dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("other", SyntaxShape::Any, "dataframe to be appended")
.switch("col", "appends in col orientation", Some('c'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Appends a dataframe as new columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"a_x".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_x".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Appends a dataframe merging at the end of columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a --col"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(3),
Value::test_int(1),
Value::test_int(3),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_int(2),
Value::test_int(4),
Value::test_int(2),
Value::test_int(4),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let other: Value = call.req(engine_state, stack, 0)?;
let axis = if call.has_flag(engine_state, stack, "col")? {
Axis::Column
} else {
Axis::Row
};
let df_other = NuDataFrame::try_from_value(other)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.append_df(&df_other, axis, call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AppendDF {})])
}
}

195
crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs
View file

@ -1,195 +0,0 @@
use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::*;
#[derive(Clone)]
pub struct CastDF;
impl Command for CastDF {
fn name(&self) -> &str {
"dfr cast"
}
fn usage(&self) -> &str {
"Cast a column to a different dtype."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.required(
"dtype",
SyntaxShape::String,
"The dtype to cast the column to",
)
.optional(
"column",
SyntaxShape::String,
"The column to cast. Required when used with a dataframe.",
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Cast a column in a dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a lazy dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a expression to a different dtype",
example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
result: None
}
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(call, column_nm, dtype, df)
} else if NuDataFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuDataFrame::try_from_value(value)?;
command_eager(call, column_nm, dtype, df)
} else {
let dtype: String = call.req(engine_state, stack, 0)?;
let dtype = str_to_dtype(&dtype, call.head)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().cast(dtype).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn df_args(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<(DataType, String), ShellError> {
let dtype = dtype_arg(engine_state, stack, call)?;
let column_nm: String =
call.opt(engine_state, stack, 1)?
.ok_or(ShellError::MissingParameter {
param_name: "column_name".into(),
span: call.head,
})?;
Ok((dtype, column_nm))
}
fn dtype_arg(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<DataType, ShellError> {
let dtype: String = call.req(engine_state, stack, 0)?;
str_to_dtype(&dtype, call.head)
}
fn command_lazy(
call: &Call,
column_nm: String,
dtype: DataType,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let column = col(&column_nm).cast(dtype);
let lazy = lazy.into_polars().with_columns(&[column]);
let lazy = NuLazyFrame::new(false, lazy);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
fn command_eager(
call: &Call,
column_nm: String,
dtype: DataType,
nu_df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mut df = nu_df.df;
let column = df
.column(&column_nm)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let _ = df
.with_column(casted)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let df = NuDataFrame::new(false, df);
Ok(PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(CastDF {})])
}
}

73
crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs
View file

@ -1,73 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ColumnsDF;
impl Command for ColumnsDF {
fn name(&self) -> &str {
"dfr columns"
}
fn usage(&self) -> &str {
"Show dataframe columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
result: Some(Value::list(
vec![Value::test_string("a"), Value::test_string("b")],
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| Value::string(*v, call.head))
.collect();
let names = Value::list(names, call.head);
Ok(PipelineData::Value(names, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ColumnsDF {})])
}
}

115
crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs
View file

@ -1,115 +0,0 @@
use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropDF;
impl Command for DropDF {
fn name(&self) -> &str {
"dfr drop"
}
fn usage(&self) -> &str {
"Creates a new dataframe by dropping the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to be dropped")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop column a",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let new_df = col_string
.first()
.ok_or_else(|| ShellError::GenericError {
error: "Empty names list".into(),
msg: "No column names were found".into(),
span: Some(col_span),
help: None,
inner: vec![],
})
.and_then(|col| {
df.as_ref()
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})?;
// If there are more columns in the drop selection list, these
// are added from the resulting dataframe
col_string
.iter()
.skip(1)
.try_fold(new_df, |new_df, col| {
new_df
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDF {})])
}
}

119
crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs
View file

@ -1,119 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::UniqueKeepStrategy;
#[derive(Clone)]
pub struct DropDuplicates;
impl Command for DropDuplicates {
fn name(&self) -> &str {
"dfr drop-duplicates"
}
fn usage(&self) -> &str {
"Drops duplicate values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop duplicates",
)
.switch("maintain", "maintain order", Some('m'))
.switch(
"last",
"keeps last duplicate value (by default keeps first)",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop duplicates",
example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(3), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(4), Value::test_int(2)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
UniqueKeepStrategy::Last
} else {
UniqueKeepStrategy::First
};
df.as_ref()
.unique(subset_slice, keep_strategy, None)
.map_err(|e| ShellError::GenericError {
error: "Error dropping duplicates".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDuplicates {})])
}
}

137
crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs
View file

@ -1,137 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropNulls;
impl Command for DropNulls {
fn name(&self) -> &str {
"dfr drop-nulls"
}
fn usage(&self) -> &str {
"Drops null values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop nulls",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "drop null values in dataframe",
example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
let res = ($df.b / $df.b);
let a = ($df | dfr with-column $res --name res);
$a | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
),
Column::new(
"res".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "drop null values in dataframe",
example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
($s / $s) | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"div_0_0".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
df.as_ref()
.drop_nulls(subset_slice)
.map_err(|e| ShellError::GenericError {
error: "Error dropping nulls".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::super::WithColumn;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})])
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs
View file

@ -1,104 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DataTypes;
impl Command for DataTypes {
fn name(&self) -> &str {
"dfr dtypes"
}
fn usage(&self) -> &str {
"Show dataframe data types."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe dtypes",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"column".to_string(),
vec![Value::test_string("a"), Value::test_string("b")],
),
Column::new(
"dtype".to_string(),
vec![Value::test_string("i64"), Value::test_string("i64")],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut dtypes: Vec<Value> = Vec::new();
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| {
let dtype = df
.as_ref()
.column(v)
.expect("using name from list of names from dataframe")
.dtype();
let dtype_str = dtype.to_string();
dtypes.push(Value::string(dtype_str, call.head));
Value::string(*v, call.head)
})
.collect();
let names_col = Column::new("column".to_string(), names);
let dtypes_col = Column::new("dtype".to_string(), dtypes);
NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DataTypes {})])
}
}

107
crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs
View file

@ -1,107 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::*, series::Series};
#[derive(Clone)]
pub struct Dummies;
impl Command for Dummies {
fn name(&self) -> &str {
"dfr dummies"
}
fn usage(&self) -> &str {
"Creates a new dataframe with dummy variables."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("drop-first", "Drop first row", Some('d'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with dummy variables from a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("a_1", &[1_u8, 0]),
Series::new("a_3", &[0_u8, 1]),
Series::new("b_2", &[1_u8, 0]),
Series::new("b_4", &[0_u8, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Create new dataframe with dummy variables from a series",
example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.to_dummies(None, drop_first)
.map_err(|e| ShellError::GenericError {
error: "Error calculating dummies".into(),
msg: e.to_string(),
span: Some(call.head),
help: Some("The only allowed column types for dummies are String or Int".into()),
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Dummies {})])
}
}

154
crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs
View file

@ -1,154 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::LazyFrame;
#[derive(Clone)]
pub struct FilterWith;
impl Command for FilterWith {
fn name(&self) -> &str {
"dfr filter-with"
}
fn usage(&self) -> &str {
"Filters dataframe using a mask or expression as reference."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"mask or expression",
SyntaxShape::Any,
"boolean mask used to filter data",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Filter dataframe using a bool mask",
example: r#"let mask = ([true false] | dfr into-df);
[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Filter dataframe using an expression",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(3)]),
Column::new("b".to_string(), vec![Value::test_int(4)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mask_value: Value = call.req(engine_state, stack, 0)?;
let mask_span = mask_value.span();
if NuExpression::can_downcast(&mask_value) {
let expression = NuExpression::try_from_value(mask_value)?;
let lazy = NuLazyFrame::new(true, df.lazy());
let lazy = lazy.apply_with_expr(expression, LazyFrame::filter);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
} else {
let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?;
let mask = mask.bool().map_err(|e| ShellError::GenericError {
error: "Error casting to bool".into(),
msg: e.to_string(),
span: Some(mask_span),
help: Some("Perhaps you want to use a series with booleans as mask".into()),
inner: vec![],
})?;
df.as_ref()
.filter(mask)
.map_err(|e| ShellError::GenericError {
error: "Error filtering dataframe".into(),
msg: e.to_string(),
span: Some(call.head),
help: Some("The only allowed column types for dummies are String or Int".into()),
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let expr: Value = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(expr)?;
let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(FilterWith {}), Box::new(ExprCol {})])
}
}

144
crates/nu-cmd-dataframe/src/dataframe/eager/first.rs
View file

@ -1,144 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct FirstDF;
impl Command for FirstDF {
fn name(&self) -> &str {
"dfr first"
}
fn usage(&self) -> &str {
"Show only the first number of rows or create a first expression"
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"rows",
SyntaxShape::Int,
"starting from the front, the number of rows to return",
)
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Return the first row of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Return the first two rows of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a first expression from a column",
example: "dfr col a | dfr first",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().first().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(1);
let res = df.as_ref().head(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(FirstDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
}
}

87
crates/nu-cmd-dataframe/src/dataframe/eager/get.rs
View file

@ -1,87 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct GetDF;
impl Command for GetDF {
fn name(&self) -> &str {
"dfr get"
}
fn usage(&self) -> &str {
"Creates dataframe with the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to sort dataframe")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns the selected column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns_string(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.select(col_string)
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetDF {})])
}
}

118
crates/nu-cmd-dataframe/src/dataframe/eager/last.rs
View file

@ -1,118 +0,0 @@
use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LastDF;
impl Command for LastDF {
fn name(&self) -> &str {
"dfr last"
}
fn usage(&self) -> &str {
"Creates new dataframe with tail rows or creates a last expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional("rows", SyntaxShape::Int, "Number of rows for tail")
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with last rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(3)]),
Column::new("b".to_string(), vec![Value::test_int(4)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a last expression from a column",
example: "dfr col a | dfr last",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().last().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(DEFAULT_ROWS);
let res = df.as_ref().tail(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(LastDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
}
}

68
crates/nu-cmd-dataframe/src/dataframe/eager/list.rs
View file

@ -1,68 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ListDF;
impl Command for ListDF {
fn name(&self) -> &str {
"dfr ls"
}
fn usage(&self) -> &str {
"Lists stored dataframes."
}
fn signature(&self) -> Signature {
Signature::build(self.name()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a new dataframe and shows it in the dataframe list",
example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
ls"#,
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let mut vals: Vec<(String, Value)> = vec![];
for overlay_frame in engine_state.active_overlays(&[]) {
for var in &overlay_frame.vars {
if let Ok(value) = stack.get_var(*var.1, call.head) {
let name = String::from_utf8_lossy(var.0).to_string();
vals.push((name, value));
}
}
}
let vals = vals
.into_iter()
.filter_map(|(name, value)| {
NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
})
.map(|(name, df)| {
Value::record(
record! {
"name" => Value::string(name, call.head),
"columns" => Value::int(df.as_ref().width() as i64, call.head),
"rows" => Value::int(df.as_ref().height() as i64, call.head),
},
call.head,
)
})
.collect::<Vec<Value>>();
let list = Value::list(vals, call.head);
Ok(list.into_pipeline_data())
}
}

248
crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs
View file

@ -1,248 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct MeltDF;
impl Command for MeltDF {
fn name(&self) -> &str {
"dfr melt"
}
fn usage(&self) -> &str {
"Unpivot a DataFrame from wide to long format."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required_named(
"columns",
SyntaxShape::Table(vec![]),
"column names for melting",
Some('c'),
)
.required_named(
"values",
SyntaxShape::Table(vec![]),
"column names used as value columns",
Some('v'),
)
.named(
"variable-name",
SyntaxShape::String,
"optional name for variable column",
Some('r'),
)
.named(
"value-name",
SyntaxShape::String,
"optional name for value column",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "melt dataframe",
example:
"[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"b".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
],
),
Column::new(
"c".to_string(),
vec![
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
],
),
Column::new(
"variable".to_string(),
vec![
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("d"),
Value::test_string("d"),
Value::test_string("d"),
],
),
Column::new(
"value".to_string(),
vec![
Value::test_string("x"),
Value::test_string("y"),
Value::test_string("z"),
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
),
], None)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let id_col: Vec<Value> = call
.get_flag(engine_state, stack, "columns")?
.expect("required value");
let val_col: Vec<Value> = call
.get_flag(engine_state, stack, "values")?
.expect("required value");
let value_name: Option<Spanned<String>> = call.get_flag(engine_state, stack, "value-name")?;
let variable_name: Option<Spanned<String>> =
call.get_flag(engine_state, stack, "variable-name")?;
let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
let mut res = df
.as_ref()
.melt(&id_col_string, &val_col_string)
.map_err(|e| ShellError::GenericError {
error: "Error calculating melt".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
if let Some(name) = &variable_name {
res.rename("variable", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
if let Some(name) = &value_name {
res.rename("value", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
fn check_column_datatypes<T: AsRef<str>>(
df: &polars::prelude::DataFrame,
cols: &[T],
col_span: Span,
) -> Result<(), ShellError> {
if cols.is_empty() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "empty column list".into(),
span: Some(col_span),
help: None,
inner: vec![],
});
}
// Checking if they are same type
if cols.len() > 1 {
for w in cols.windows(2) {
let l_series = df
.column(w[0].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
let r_series = df
.column(w[1].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
if l_series.dtype() != r_series.dtype() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "found different column types in list".into(),
span: Some(col_span),
help: Some(format!(
"datatypes {} and {} are incompatible",
l_series.dtype(),
r_series.dtype()
)),
inner: vec![],
});
}
}
}
Ok(())
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(MeltDF {})])
}
}

114
crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs
View file

@ -1,114 +0,0 @@
mod append;
mod cast;
mod columns;
mod drop;
mod drop_duplicates;
mod drop_nulls;
mod dtypes;
mod dummies;
mod filter_with;
mod first;
mod get;
mod last;
mod list;
mod melt;
mod open;
mod query_df;
mod rename;
mod sample;
mod schema;
mod shape;
mod slice;
mod sql_context;
mod sql_expr;
mod summary;
mod take;
mod to_arrow;
mod to_avro;
mod to_csv;
mod to_df;
mod to_json_lines;
mod to_nu;
mod to_parquet;
mod with_column;
use nu_protocol::engine::StateWorkingSet;
pub use self::open::OpenDataFrame;
pub use append::AppendDF;
pub use cast::CastDF;
pub use columns::ColumnsDF;
pub use drop::DropDF;
pub use drop_duplicates::DropDuplicates;
pub use drop_nulls::DropNulls;
pub use dtypes::DataTypes;
pub use dummies::Dummies;
pub use filter_with::FilterWith;
pub use first::FirstDF;
pub use get::GetDF;
pub use last::LastDF;
pub use list::ListDF;
pub use melt::MeltDF;
pub use query_df::QueryDf;
pub use rename::RenameDF;
pub use sample::SampleDF;
pub use schema::SchemaDF;
pub use shape::ShapeDF;
pub use slice::SliceDF;
pub use sql_context::SQLContext;
pub use summary::Summary;
pub use take::TakeDF;
pub use to_arrow::ToArrow;
pub use to_avro::ToAvro;
pub use to_csv::ToCSV;
pub use to_df::ToDataFrame;
pub use to_json_lines::ToJsonLines;
pub use to_nu::ToNu;
pub use to_parquet::ToParquet;
pub use with_column::WithColumn;
pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
( $( $command:expr ),* ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
// Dataframe commands
bind_command!(
AppendDF,
CastDF,
ColumnsDF,
DataTypes,
Summary,
DropDF,
DropDuplicates,
DropNulls,
Dummies,
FilterWith,
FirstDF,
GetDF,
LastDF,
ListDF,
MeltDF,
OpenDataFrame,
QueryDf,
RenameDF,
SampleDF,
SchemaDF,
ShapeDF,
SliceDF,
TakeDF,
ToArrow,
ToAvro,
ToCSV,
ToDataFrame,
ToNu,
ToParquet,
ToJsonLines,
WithColumn
);
}

518
crates/nu-cmd-dataframe/src/dataframe/eager/open.rs
View file

@ -1,518 +0,0 @@
use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
use nu_engine::command_prelude::*;
use polars::prelude::{
CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
};
use polars_io::{avro::AvroReader, HiveOptions};
use std::{fs::File, io::BufReader, path::PathBuf};
#[derive(Clone)]
pub struct OpenDataFrame;
impl Command for OpenDataFrame {
fn name(&self) -> &str {
"dfr open"
}
fn usage(&self) -> &str {
"Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"file",
SyntaxShape::Filepath,
"file path to load values from",
)
.switch("lazy", "creates a lazy dataframe", Some('l'))
.named(
"type",
SyntaxShape::String,
"File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension",
Some('t'),
)
.named(
"delimiter",
SyntaxShape::String,
"file delimiter character. CSV file",
Some('d'),
)
.switch(
"no-header",
"Indicates if file doesn't have header. CSV file",
None,
)
.named(
"infer-schema",
SyntaxShape::Number,
"Number of rows to infer the schema of the file. CSV file",
None,
)
.named(
"skip-rows",
SyntaxShape::Number,
"Number of rows to skip from file. CSV file",
None,
)
.named(
"columns",
SyntaxShape::List(Box::new(SyntaxShape::String)),
"Columns to be selected from csv file. CSV and Parquet file",
None,
)
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s')
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Takes a file name and creates a dataframe",
example: "dfr open test.csv",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<PipelineData, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
let type_id = match &type_option {
Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
None => file.item.extension().map(|e| {
(
e.to_string_lossy().into_owned(),
"Invalid extension",
file.span,
)
}),
};
match type_id {
Some((e, msg, blamed)) => match e.as_str() {
"csv" | "tsv" => from_csv(engine_state, stack, call),
"parquet" | "parq" => from_parquet(engine_state, stack, call),
"ipc" | "arrow" => from_ipc(engine_state, stack, call),
"json" => from_json(engine_state, stack, call),
"jsonl" => from_jsonl(engine_state, stack, call),
"avro" => from_avro(engine_state, stack, call),
_ => Err(ShellError::FileNotFoundCustom {
msg: format!(
"{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro"
),
span: blamed,
}),
},
None => Err(ShellError::FileNotFoundCustom {
msg: "File without extension".into(),
span: file.span,
}),
}
.map(|value| PipelineData::Value(value, None))
}
fn from_parquet(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsParquet {
n_rows: None,
cache: true,
parallel: ParallelStrategy::Auto,
rechunk: false,
row_index: None,
low_memory: false,
cloud_options: None,
use_statistics: false,
hive_options: HiveOptions::default(),
};
let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = ParquetReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_avro(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = AvroReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Avro reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_ipc(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsIpc {
n_rows: None,
cache: true,
rechunk: false,
row_index: None,
memory_map: true,
cloud_options: None,
};
let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = IpcReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_json(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_jsonl(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader)
.with_json_format(JsonFormat::JsonLines)
.infer_schema_len(infer_schema);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json lines reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_csv(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let csv_reader = LazyCsvReader::new(file);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.with_infer_schema_length(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let df: NuLazyFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let csv_reader = CsvReader::from_path(&file.item)
.map_err(|e| ShellError::GenericError {
error: "Error creating CSV reader".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?
.with_encoding(CsvEncoding::LossyUtf8);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.infer_schema(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let csv_reader = match columns {
None => csv_reader,
Some(columns) => csv_reader.with_columns(Some(columns)),
};
let df: NuDataFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs
View file

@ -1,104 +0,0 @@
use crate::dataframe::{
eager::SQLContext,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
// attribution:
// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
// maybe we should just use the crate at some point but it's not published yet.
// https://github.com/pola-rs/polars/tree/master/polars-sql
#[derive(Clone)]
pub struct QueryDf;
impl Command for QueryDf {
fn name(&self) -> &str {
"dfr query"
}
fn usage(&self) -> &str {
"Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("sql", SyntaxShape::String, "sql query")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn search_terms(&self) -> Vec<&str> {
vec!["dataframe", "sql", "search"]
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Query dataframe using SQL",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let sql_query: String = call.req(engine_state, stack, 0)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut ctx = SQLContext::new();
ctx.register("df", &df.df);
let df_sql = ctx
.execute(&sql_query)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let lazy = NuLazyFrame::new(false, df_sql);
let eager = lazy.collect(call.head)?;
let value = Value::custom(Box::new(eager), call.head);
Ok(PipelineData::Value(value, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(QueryDf {})])
}
}

185
crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs
View file

@ -1,185 +0,0 @@
use crate::dataframe::{
utils::extract_strings,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct RenameDF;
impl Command for RenameDF {
fn name(&self) -> &str {
"dfr rename"
}
fn usage(&self) -> &str {
"Rename a dataframe column."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"columns",
SyntaxShape::Any,
"Column(s) to be renamed. A string or list of strings",
)
.required(
"new names",
SyntaxShape::Any,
"New names for the selected column(s). A string or list of strings",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Renames a series",
example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"new_name".to_string(),
vec![
Value::test_int(5),
Value::test_int(6),
Value::test_int(7),
Value::test_int(8),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames a dataframe column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames two dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_new".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
mut df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
for (from, to) in columns.iter().zip(new_names.iter()) {
df.as_mut()
.rename(from, to)
.map_err(|e| ShellError::GenericError {
error: "Error renaming".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(df.into_value(call.head), None))
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
if columns.len() != new_names.len() {
let value: Value = call.req(engine_state, stack, 1)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "New name list has different size to column list".into(),
span: value.span(),
});
}
let lazy = lazy.into_polars();
let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(RenameDF {})])
}
}

127
crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs
View file

@ -1,127 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::NamedFrom, series::Series};
#[derive(Clone)]
pub struct SampleDF;
impl Command for SampleDF {
fn name(&self) -> &str {
"dfr sample"
}
fn usage(&self) -> &str {
"Create sample dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"n-rows",
SyntaxShape::Int,
"number of rows to be taken from dataframe",
Some('n'),
)
.named(
"fraction",
SyntaxShape::Number,
"fraction of dataframe to be taken",
Some('f'),
)
.named(
"seed",
SyntaxShape::Number,
"seed for the selection",
Some('s'),
)
.switch("replace", "sample with replace", Some('e'))
.switch("shuffle", "shuffle sample", Some('u'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Sample rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1",
result: None, // No expected value because sampling is random
},
Example {
description: "Shows sample row using fraction and replace",
example:
"[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace",
result: None, // No expected value because sampling is random
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let rows: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "n-rows")?;
let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
let seed: Option<u64> = call
.get_flag::<i64>(engine_state, stack, "seed")?
.map(|val| val as u64);
let replace: bool = call.has_flag(engine_state, stack, "replace")?;
let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
match (rows, fraction) {
(Some(rows), None) => df
.as_ref()
.sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(rows.span),
help: None,
inner: vec![],
}),
(None, Some(frac)) => df
.as_ref()
.sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(frac.span),
help: None,
inner: vec![],
}),
(Some(_), Some(_)) => Err(ShellError::GenericError {
error: "Incompatible flags".into(),
msg: "Only one selection criterion allowed".into(),
span: Some(call.head),
help: None,
inner: vec![],
}),
(None, None) => Err(ShellError::GenericError {
error: "No selection".into(),
msg: "No selection criterion was found".into(),
span: Some(call.head),
help: Some("Perhaps you want to use the flag -n or -f".into()),
inner: vec![],
}),
}
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}

112
crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs
View file

@ -1,112 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SchemaDF;
impl Command for SchemaDF {
fn name(&self) -> &str {
"dfr schema"
}
fn usage(&self) -> &str {
"Show schema for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("datatype-list", "creates a lazy dataframe", Some('l'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe schema",
example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
result: Some(Value::record(
record! {
"a" => Value::string("i64", Span::test_data()),
"b" => Value::string("str", Span::test_data()),
},
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
if call.has_flag(engine_state, stack, "datatype-list")? {
Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
} else {
command(engine_state, stack, call, input)
}
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let schema = df.schema();
let value: Value = schema.into();
Ok(PipelineData::Value(value, None))
}
fn datatype_list(span: Span) -> Value {
let types: Vec<Value> = [
("null", ""),
("bool", ""),
("u8", ""),
("u16", ""),
("u32", ""),
("u64", ""),
("i8", ""),
("i16", ""),
("i32", ""),
("i64", ""),
("f32", ""),
("f64", ""),
("str", ""),
("binary", ""),
("date", ""),
("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
("time", ""),
("object", ""),
("unknown", ""),
("list<dtype>", ""),
]
.iter()
.map(|(dtype, note)| {
Value::record(record! {
"dtype" => Value::string(*dtype, span),
"note" => Value::string(*note, span),
},
span)
})
.collect();
Value::list(types, span)
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SchemaDF {})])
}
}

82
crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs
View file

@ -1,82 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ShapeDF;
impl Command for ShapeDF {
fn name(&self) -> &str {
"dfr shape"
}
fn usage(&self) -> &str {
"Shows column and row size for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Shows row and column shape",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("rows".to_string(), vec![Value::test_int(2)]),
Column::new("columns".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let rows = Value::int(df.as_ref().height() as i64, call.head);
let cols = Value::int(df.as_ref().width() as i64, call.head);
let rows_col = Column::new("rows".to_string(), vec![rows]);
let cols_col = Column::new("columns".to_string(), vec![cols]);
NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ShapeDF {})])
}
}

84
crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs
View file

@ -1,84 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SliceDF;
impl Command for SliceDF {
fn name(&self) -> &str {
"dfr slice"
}
fn usage(&self) -> &str {
"Creates new dataframe from a slice of rows."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("offset", SyntaxShape::Int, "start of slice")
.required("size", SyntaxShape::Int, "size of slice")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Create new dataframe from a slice of the rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let offset: i64 = call.req(engine_state, stack, 0)?;
let size: usize = call.req(engine_state, stack, 1)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let res = df.as_ref().slice(offset, size);
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SliceDF {})])
}
}

228
crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs
View file

@ -1,228 +0,0 @@
use crate::dataframe::eager::sql_expr::parse_sql_expr;
use polars::error::{ErrString, PolarsError};
use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame};
use sqlparser::ast::{
Expr as SqlExpr, GroupByExpr, Select, SelectItem, SetExpr, Statement, TableFactor,
Value as SQLValue,
};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
use std::collections::HashMap;
#[derive(Default)]
pub struct SQLContext {
table_map: HashMap<String, LazyFrame>,
dialect: GenericDialect,
}
impl SQLContext {
pub fn new() -> Self {
Self {
table_map: HashMap::new(),
dialect: GenericDialect,
}
}
pub fn register(&mut self, name: &str, df: &DataFrame) {
self.table_map.insert(name.to_owned(), df.clone().lazy());
}
fn execute_select(&self, select_stmt: &Select) -> Result<LazyFrame, PolarsError> {
// Determine involved dataframe
// Implicit join require some more work in query parsers, Explicit join are preferred for now.
let tbl = select_stmt.from.first().ok_or_else(|| {
PolarsError::ComputeError(ErrString::from("No table found in select statement"))
})?;
let mut alias_map = HashMap::new();
let tbl_name = match &tbl.relation {
TableFactor::Table { name, alias, .. } => {
let tbl_name = name
.0
.first()
.ok_or_else(|| {
PolarsError::ComputeError(ErrString::from(
"No table found in select statement",
))
})?
.value
.to_string();
if self.table_map.contains_key(&tbl_name) {
if let Some(alias) = alias {
alias_map.insert(alias.name.value.clone(), tbl_name.to_owned());
};
tbl_name
} else {
return Err(PolarsError::ComputeError(
format!("Table name {tbl_name} was not found").into(),
));
}
}
// Support bare table, optional with alias for now
_ => return Err(PolarsError::ComputeError("Not implemented".into())),
};
let df = &self.table_map[&tbl_name];
let mut raw_projection_before_alias: HashMap<String, usize> = HashMap::new();
let mut contain_wildcard = false;
// Filter Expression
let df = match select_stmt.selection.as_ref() {
Some(expr) => {
let filter_expression = parse_sql_expr(expr)?;
df.clone().filter(filter_expression)
}
None => df.clone(),
};
// Column Projections
let projection = select_stmt
.projection
.iter()
.enumerate()
.map(|(i, select_item)| {
Ok(match select_item {
SelectItem::UnnamedExpr(expr) => {
let expr = parse_sql_expr(expr)?;
raw_projection_before_alias.insert(format!("{expr:?}"), i);
expr
}
SelectItem::ExprWithAlias { expr, alias } => {
let expr = parse_sql_expr(expr)?;
raw_projection_before_alias.insert(format!("{expr:?}"), i);
expr.alias(&alias.value)
}
SelectItem::QualifiedWildcard(_, _) | SelectItem::Wildcard(_) => {
contain_wildcard = true;
col("*")
}
})
})
.collect::<Result<Vec<_>, PolarsError>>()?;
// Check for group by
// After projection since there might be number.
let group_by = match &select_stmt.group_by {
GroupByExpr::All =>
Err(
PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported, not all".into())
)?,
GroupByExpr::Expressions(expressions) => expressions
}
.iter()
.map(
|e|match e {
SqlExpr::Value(SQLValue::Number(idx, _)) => {
let idx = match idx.parse::<usize>() {
Ok(0)| Err(_) => Err(
PolarsError::ComputeError(
format!("Group-By Error: Only positive number or expression are supported, got {idx}").into()
)),
Ok(idx) => Ok(idx)
}?;
Ok(projection[idx].clone())
}
SqlExpr::Value(_) => Err(
PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported".into())
),
_ => parse_sql_expr(e)
}
)
.collect::<Result<Vec<_>, PolarsError>>()?;
let df = if group_by.is_empty() {
df.select(projection)
} else {
// check groupby and projection due to difference between SQL and polars
// Return error on wild card, shouldn't process this
if contain_wildcard {
return Err(PolarsError::ComputeError(
"Group-By Error: Can't process wildcard in group-by".into(),
));
}
// Default polars group by will have group by columns at the front
// need some container to contain position of group by columns and its position
// at the final agg projection, check the schema for the existence of group by column
// and its projections columns, keeping the original index
let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by
.iter()
.map(|expr| raw_projection_before_alias.get(&format!("{expr:?}")))
.enumerate()
.filter(|(_, proj_p)| proj_p.is_some())
.map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p)))
.unzip();
let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection
.iter()
.enumerate()
.filter(|(i, _)| !exclude_expr.contains(i))
.enumerate()
.map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len())))
.unzip();
let agg_df = df.group_by(group_by).agg(agg_projection);
let mut final_proj_pos = groupby_pos
.into_iter()
.chain(agg_proj_pos)
.collect::<Vec<_>>();
final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb));
let final_proj = final_proj_pos
.into_iter()
.map(|(_, shm_p)| {
col(agg_df
.clone()
// FIXME: had to do this mess to get get_index to work, not sure why. need help
.collect()
.unwrap_or_default()
.schema()
.get_at_index(shm_p)
.unwrap_or((&"".into(), &DataType::Null))
.0)
})
.collect::<Vec<_>>();
agg_df.select(final_proj)
};
Ok(df)
}
pub fn execute(&self, query: &str) -> Result<LazyFrame, PolarsError> {
let ast = Parser::parse_sql(&self.dialect, query)
.map_err(|e| PolarsError::ComputeError(format!("{e:?}").into()))?;
if ast.len() != 1 {
Err(PolarsError::ComputeError(
"One and only one statement at a time please".into(),
))
} else {
let ast = ast
.first()
.ok_or_else(|| PolarsError::ComputeError(ErrString::from("No statement found")))?;
Ok(match ast {
Statement::Query(query) => {
let rs = match &*query.body {
SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?,
_ => {
return Err(PolarsError::ComputeError(
"INSERT, UPDATE is not supported for polars".into(),
))
}
};
match &query.limit {
Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => {
let nrow = nrow.parse().map_err(|err| {
PolarsError::ComputeError(
format!("Conversion Error: {err:?}").into(),
)
})?;
rs.limit(nrow)
}
None => rs,
_ => {
return Err(PolarsError::ComputeError(
"Only support number argument to LIMIT clause".into(),
))
}
}
}
_ => {
return Err(PolarsError::ComputeError(
format!("Statement type {ast:?} is not supported").into(),
))
}
})
}
}
}

200
crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs
View file

@ -1,200 +0,0 @@
use polars::error::PolarsError;
use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit};
use sqlparser::ast::{
ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType,
Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType,
};
fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
Ok(match data_type {
SQLDataType::Char(_)
| SQLDataType::Varchar(_)
| SQLDataType::Uuid
| SQLDataType::Clob(_)
| SQLDataType::Text
| SQLDataType::String(_) => DataType::String,
SQLDataType::Float(_) => DataType::Float32,
SQLDataType::Real => DataType::Float32,
SQLDataType::Double => DataType::Float64,
SQLDataType::TinyInt(_) => DataType::Int8,
SQLDataType::UnsignedTinyInt(_) => DataType::UInt8,
SQLDataType::SmallInt(_) => DataType::Int16,
SQLDataType::UnsignedSmallInt(_) => DataType::UInt16,
SQLDataType::Int(_) => DataType::Int32,
SQLDataType::UnsignedInt(_) => DataType::UInt32,
SQLDataType::BigInt(_) => DataType::Int64,
SQLDataType::UnsignedBigInt(_) => DataType::UInt64,
SQLDataType::Boolean => DataType::Boolean,
SQLDataType::Date => DataType::Date,
SQLDataType::Time(_, _) => DataType::Time,
SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None),
SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds),
SQLDataType::Array(array_type_def) => match array_type_def {
ArrayElemTypeDef::AngleBracket(inner_type)
| ArrayElemTypeDef::SquareBracket(inner_type) => {
DataType::List(Box::new(map_sql_polars_datatype(inner_type)?))
}
_ => {
return Err(PolarsError::ComputeError(
"SQL Datatype Array(None) was not supported in polars-sql yet!".into(),
))
}
},
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn cast_(expr: Expr, data_type: &SQLDataType) -> Result<Expr> {
let polars_type = map_sql_polars_datatype(data_type)?;
Ok(expr.cast(polars_type))
}
fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
Ok(match op {
SQLBinaryOperator::Plus => left + right,
SQLBinaryOperator::Minus => left - right,
SQLBinaryOperator::Multiply => left * right,
SQLBinaryOperator::Divide => left / right,
SQLBinaryOperator::Modulo => left % right,
SQLBinaryOperator::StringConcat => {
left.cast(DataType::String) + right.cast(DataType::String)
}
SQLBinaryOperator::Gt => left.gt(right),
SQLBinaryOperator::Lt => left.lt(right),
SQLBinaryOperator::GtEq => left.gt_eq(right),
SQLBinaryOperator::LtEq => left.lt_eq(right),
SQLBinaryOperator::Eq => left.eq(right),
SQLBinaryOperator::NotEq => left.eq(right).not(),
SQLBinaryOperator::And => left.and(right),
SQLBinaryOperator::Or => left.or(right),
SQLBinaryOperator::Xor => left.xor(right),
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn literal_expr(value: &SqlValue) -> Result<Expr> {
Ok(match value {
SqlValue::Number(s, _) => {
// Check for existence of decimal separator dot
if s.contains('.') {
s.parse::<f64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
} else {
s.parse::<i64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
}?
}
SqlValue::SingleQuotedString(s) => lit(s.clone()),
SqlValue::NationalStringLiteral(s) => lit(s.clone()),
SqlValue::HexStringLiteral(s) => lit(s.clone()),
SqlValue::DoubleQuotedString(s) => lit(s.clone()),
SqlValue::Boolean(b) => lit(*b),
SqlValue::Null => Expr::Literal(LiteralValue::Null),
_ => {
return Err(PolarsError::ComputeError(
format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(),
))
}
})
}
pub fn parse_sql_expr(expr: &SqlExpr) -> Result<Expr> {
Ok(match expr {
SqlExpr::Identifier(e) => col(&e.value),
SqlExpr::BinaryOp { left, op, right } => {
let left = parse_sql_expr(left)?;
let right = parse_sql_expr(right)?;
binary_op_(left, right, op)?
}
SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?,
SqlExpr::Cast {
expr,
data_type,
format: _,
} => cast_(parse_sql_expr(expr)?, data_type)?,
SqlExpr::Nested(expr) => parse_sql_expr(expr)?,
SqlExpr::Value(value) => literal_expr(value)?,
_ => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result<Expr> {
Ok(match &window_type {
Some(wtype) => match wtype {
WindowType::WindowSpec(window_spec) => {
// Process for simple window specification, partition by first
let partition_by = window_spec
.partition_by
.iter()
.map(parse_sql_expr)
.collect::<Result<Vec<_>>>()?;
expr.over(partition_by)
// Order by and Row range may not be supported at the moment
}
// TODO: make NamedWindow work
WindowType::NamedWindow(_named) => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
},
None => expr,
})
}
fn parse_sql_function(sql_function: &SQLFunction) -> Result<Expr> {
use sqlparser::ast::{FunctionArg, FunctionArgExpr};
// Function name mostly do not have name space, so it mostly take the first args
let function_name = sql_function.name.0[0].value.to_ascii_lowercase();
let args = sql_function
.args
.iter()
.map(|arg| match arg {
FunctionArg::Named { arg, .. } => arg,
FunctionArg::Unnamed(arg) => arg,
})
.collect::<Vec<_>>();
Ok(
match (
function_name.as_str(),
args.as_slice(),
sql_function.distinct,
) {
("sum", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum()
}
("count", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count()
}
("count", [FunctionArgExpr::Expr(expr)], true) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique()
}
// Special case for wildcard args to count function.
("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(),
_ => {
return Err(PolarsError::ComputeError(
format!(
"Function {function_name:?} with args {args:?} was not supported in polars-sql yet!"
)
.into(),
))
}
},
)
}

279
crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs
View file

@ -1,279 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::{
chunked_array::ChunkedArray,
prelude::{
AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
QuantileInterpolOptions, Series, StringType,
},
};
#[derive(Clone)]
pub struct Summary;
impl Command for Summary {
fn name(&self) -> &str {
"dfr summary"
}
fn usage(&self) -> &str {
"For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.category(Category::Custom("dataframe".into()))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.named(
"quantiles",
SyntaxShape::Table(vec![]),
"provide optional quantiles",
Some('q'),
)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "list dataframe descriptives",
example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"descriptor".to_string(),
vec![
Value::test_string("count"),
Value::test_string("sum"),
Value::test_string("mean"),
Value::test_string("median"),
Value::test_string("std"),
Value::test_string("min"),
Value::test_string("25%"),
Value::test_string("50%"),
Value::test_string("75%"),
Value::test_string("max"),
],
),
Column::new(
"a (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
Column::new(
"b (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let quantiles: Option<Vec<Value>> = call.get_flag(engine_state, stack, "quantiles")?;
let quantiles = quantiles.map(|values| {
values
.iter()
.map(|value| {
let span = value.span();
match value {
Value::Float { val, .. } => {
if (&0.0..=&1.0).contains(&val) {
Ok(*val)
} else {
Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be between 0 and 1".into(),
span: Some(span),
help: None,
inner: vec![],
})
}
}
Value::Error { error, .. } => Err(*error.clone()),
_ => Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be a float".into(),
span: Some(span),
help: None,
inner: vec![],
}),
}
})
.collect::<Result<Vec<f64>, ShellError>>()
});
let quantiles = match quantiles {
Some(quantiles) => quantiles?,
None => vec![0.25, 0.50, 0.75],
};
let mut quantiles_labels = quantiles
.iter()
.map(|q| Some(format!("{}%", q * 100.0)))
.collect::<Vec<Option<String>>>();
let mut labels = vec![
Some("count".to_string()),
Some("sum".to_string()),
Some("mean".to_string()),
Some("median".to_string()),
Some("std".to_string()),
Some("min".to_string()),
];
labels.append(&mut quantiles_labels);
labels.push(Some("max".to_string()));
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
let head = std::iter::once(names);
let tail = df
.as_ref()
.get_columns()
.iter()
.filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
.map(|col| {
let count = col.len() as f64;
let sum = col.sum_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mean = match col.mean_as_series().get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
};
let median = match col.median_as_series() {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let std = match col.std_as_series(0) {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let min = col.min_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut quantiles = quantiles
.clone()
.into_iter()
.map(|q| {
col.quantile_as_series(q, QuantileInterpolOptions::default())
.ok()
.and_then(|ca| ca.cast(&DataType::Float64).ok())
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
})
.collect::<Vec<Option<f64>>>();
let max = col.max_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut descriptors = vec![Some(count), sum, mean, median, std, min];
descriptors.append(&mut quantiles);
descriptors.push(max);
let name = format!("{} ({})", col.name(), col.dtype());
ChunkedArray::<Float64Type>::from_slice_options(&name, &descriptors).into_series()
});
let res = head.chain(tail).collect::<Vec<Series>>();
DataFrame::new(res)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Summary {})])
}
}

148
crates/nu-cmd-dataframe/src/dataframe/eager/take.rs
View file

@ -1,148 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::DataType;
#[derive(Clone)]
pub struct TakeDF;
impl Command for TakeDF {
fn name(&self) -> &str {
"dfr take"
}
fn usage(&self) -> &str {
"Creates new dataframe using the given indices."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"indices",
SyntaxShape::Any,
"list of indices used to take data",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Takes selected rows from dataframe",
example: r#"let df = ([[a b]; [4 1] [5 2] [4 3]] | dfr into-df);
let indices = ([0 2] | dfr into-df);
$df | dfr take $indices"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(4), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes selected rows from series",
example: r#"let series = ([4 1 5 2 4 3] | dfr into-df);
let indices = ([0 2] | dfr into-df);
$series | dfr take $indices"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(4), Value::test_int(5)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let index_value: Value = call.req(engine_state, stack, 0)?;
let index_span = index_value.span();
let index = NuDataFrame::try_from_value(index_value)?.as_series(index_span)?;
let casted = match index.dtype() {
DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index
.cast(&DataType::UInt32)
.map_err(|e| ShellError::GenericError {
error: "Error casting index list".into(),
msg: e.to_string(),
span: Some(index_span),
help: None,
inner: vec![],
}),
_ => Err(ShellError::GenericError {
error: "Incorrect type".into(),
msg: "Series with incorrect type".into(),
span: Some(call.head),
help: Some("Consider using a Series with type int type".into()),
inner: vec![],
}),
}?;
let indices = casted.u32().map_err(|e| ShellError::GenericError {
error: "Error casting index list".into(),
msg: e.to_string(),
span: Some(index_span),
help: None,
inner: vec![],
})?;
NuDataFrame::try_from_pipeline(input, call.head).and_then(|df| {
df.as_ref()
.take(indices)
.map_err(|e| ShellError::GenericError {
error: "Error taking values".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
})
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(TakeDF {})])
}
}

79
crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs
View file

@ -1,79 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{IpcWriter, SerWriter};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToArrow;
impl Command for ToArrow {
fn name(&self) -> &str {
"dfr to-arrow"
}
fn usage(&self) -> &str {
"Saves dataframe to arrow file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to arrow file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-arrow test.arrow",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
IpcWriter::new(&mut file)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

109
crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs
View file

@ -1,109 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars_io::{
avro::{AvroCompression, AvroWriter},
SerWriter,
};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToAvro;
impl Command for ToAvro {
fn name(&self) -> &str {
"dfr to-avro"
}
fn usage(&self) -> &str {
"Saves dataframe to avro file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"compression",
SyntaxShape::String,
"use compression, supports deflate or snappy",
Some('c'),
)
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to avro file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-avro test.avro",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn get_compression(call: &Call) -> Result<Option<AvroCompression>, ShellError> {
if let Some((compression, span)) = call
.get_flag_expr("compression")
.and_then(|e| e.as_string().map(|s| (s, e.span)))
{
match compression.as_ref() {
"snappy" => Ok(Some(AvroCompression::Snappy)),
"deflate" => Ok(Some(AvroCompression::Deflate)),
_ => Err(ShellError::IncorrectValue {
msg: "compression must be one of deflate or snappy".to_string(),
val_span: span,
call_span: span,
}),
}
} else {
Ok(None)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let compression = get_compression(call)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
AvroWriter::new(file)
.with_compression(compression)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

125
crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs
View file

@ -1,125 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{CsvWriter, SerWriter};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToCSV;
impl Command for ToCSV {
fn name(&self) -> &str {
"dfr to-csv"
}
fn usage(&self) -> &str {
"Saves dataframe to CSV file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.named(
"delimiter",
SyntaxShape::String,
"file delimiter character",
Some('d'),
)
.switch("no-header", "Indicates if file doesn't have header", None)
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Saves dataframe to CSV file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv",
result: None,
},
Example {
description: "Saves dataframe to CSV file using other delimiter",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv --delimiter '|'",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let writer = CsvWriter::new(&mut file);
let writer = if no_header {
writer.include_header(false)
} else {
writer.include_header(true)
};
let mut writer = match delimiter {
None => writer,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one char".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
writer.with_separator(delimiter)
}
}
};
writer
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error writing to file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

189
crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs
View file

@ -1,189 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuSchema};
use nu_engine::command_prelude::*;
use polars::prelude::*;
#[derive(Clone)]
pub struct ToDataFrame;
impl Command for ToDataFrame {
fn name(&self) -> &str {
"dfr into-df"
}
fn usage(&self) -> &str {
"Converts a list, table or record into a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s'),
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Takes a dictionary and creates a dataframe",
example: "[[a b];[1 2] [3 4]] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list of tables and creates a dataframe",
example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"0".to_string(),
vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
),
Column::new(
"1".to_string(),
vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
),
Column::new(
"2".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list and creates a dataframe",
example: "[a b c] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list of booleans and creates a dataframe",
example: "[true true false] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Convert to a dataframe and provide a schema",
example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list<u64>}, c: list<str>}",
result: Some(
NuDataFrame::try_from_series(vec![
Series::new("a", &[1u8]),
{
let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]);
let vals = vec![AnyValue::StructOwned(
Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1];
Series::from_any_values_and_dtype("b", &vals, &dtype, false)
.expect("Struct series should not fail")
},
{
let dtype = DataType::List(Box::new(DataType::String));
let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))];
Series::from_any_values_and_dtype("c", &vals, &dtype, false)
.expect("List series should not fail")
}
], Span::test_data())
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Convert to a dataframe and provide a schema that adds a new column",
example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#,
result: Some(NuDataFrame::try_from_series(vec![
Series::new("a", [1u8, 2]),
Series::new("b", ["foo", "bar"]),
Series::new("c", [3i64, 3]),
], Span::test_data())
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?;
Ok(PipelineData::Value(
NuDataFrame::into_value(df, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ToDataFrame {})])
}
}

80
crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs
View file

@ -1,80 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{JsonWriter, SerWriter};
use std::{fs::File, io::BufWriter, path::PathBuf};
#[derive(Clone)]
pub struct ToJsonLines;
impl Command for ToJsonLines {
fn name(&self) -> &str {
"dfr to-jsonl"
}
fn usage(&self) -> &str {
"Saves dataframe to a JSON lines file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to JSON lines file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-jsonl test.jsonl",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let buf_writer = BufWriter::new(file);
JsonWriter::new(buf_writer)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

136
crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs
View file

@ -1,136 +0,0 @@
use crate::dataframe::values::{NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ToNu;
impl Command for ToNu {
fn name(&self) -> &str {
"dfr into-nu"
}
fn usage(&self) -> &str {
"Converts a dataframe or an expression into into nushell value for access and exploration."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"rows",
SyntaxShape::Number,
"number of rows to be shown",
Some('n'),
)
.switch("tail", "shows tail rows", Some('t'))
.input_output_types(vec![
(Type::Custom("expression".into()), Type::Any),
(Type::Custom("dataframe".into()), Type::table()),
])
//.input_output_type(Type::Any, Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
let rec_1 = Value::test_record(record! {
"index" => Value::test_int(0),
"a" => Value::test_int(1),
"b" => Value::test_int(2),
});
let rec_2 = Value::test_record(record! {
"index" => Value::test_int(1),
"a" => Value::test_int(3),
"b" => Value::test_int(4),
});
let rec_3 = Value::test_record(record! {
"index" => Value::test_int(2),
"a" => Value::test_int(3),
"b" => Value::test_int(4),
});
vec![
Example {
description: "Shows head rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-nu",
result: Some(Value::list(vec![rec_1, rec_2], Span::test_data())),
},
Example {
description: "Shows tail rows from dataframe",
example: "[[a b]; [1 2] [5 6] [3 4]] | dfr into-df | dfr into-nu --tail --rows 1",
result: Some(Value::list(vec![rec_3], Span::test_data())),
},
Example {
description: "Convert a col expression into a nushell value",
example: "dfr col a | dfr into-nu",
result: Some(Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
})),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
dataframe_command(engine_state, stack, call, value)
} else {
expression_command(call, value)
}
}
}
fn dataframe_command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: Value,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.get_flag(engine_state, stack, "rows")?;
let tail: bool = call.has_flag(engine_state, stack, "tail")?;
let df = NuDataFrame::try_from_value(input)?;
let values = if tail {
df.tail(rows, call.head)?
} else {
// if rows is specified, return those rows, otherwise return everything
if rows.is_some() {
df.head(rows, call.head)?
} else {
df.head(Some(df.height()), call.head)?
}
};
let value = Value::list(values, call.head);
Ok(PipelineData::Value(value, None))
}
fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_value(input)?;
let value = expr.to_value(call.head)?;
Ok(PipelineData::Value(value, None))
}
#[cfg(test)]
mod test {
use super::super::super::expressions::ExprCol;
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples_dataframe_input() {
test_dataframe(vec![Box::new(ToNu {})])
}
#[test]
fn test_examples_expression_input() {
test_dataframe(vec![Box::new(ToNu {}), Box::new(ExprCol {})])
}
}

79
crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs
View file

@ -1,79 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::ParquetWriter;
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToParquet;
impl Command for ToParquet {
fn name(&self) -> &str {
"dfr to-parquet"
}
fn usage(&self) -> &str {
"Saves dataframe to parquet file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to parquet file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-parquet test.parquet",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
ParquetWriter::new(file)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

202
crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs
View file

@ -1,202 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct WithColumn;
impl Command for WithColumn {
fn name(&self) -> &str {
"dfr with-column"
}
fn usage(&self) -> &str {
"Adds a series to the dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named("name", SyntaxShape::String, "new column name", Some('n'))
.rest(
"series or expressions",
SyntaxShape::Any,
"series to be added or expressions used to define the new columns",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Adds a series to the dataframe",
example: r#"[[a b]; [1 2] [3 4]]
| dfr into-df
| dfr with-column ([5 6] | dfr into-df) --name c"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(5), Value::test_int(6)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Adds a series to the dataframe",
example: r#"[[a b]; [1 2] [3 4]]
| dfr into-lazy
| dfr with-column [
((dfr col a) * 2 | dfr as "c")
((dfr col a) * 3 | dfr as "d")
]
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(2), Value::test_int(6)],
),
Column::new(
"d".to_string(),
vec![Value::test_int(3), Value::test_int(9)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
} else {
Err(ShellError::CantConvert {
to_type: "lazy or eager dataframe".into(),
from_type: value.get_type().to_string(),
span: value.span(),
help: None,
})
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
mut df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let new_column: Value = call.req(engine_state, stack, 0)?;
let column_span = new_column.span();
if NuExpression::can_downcast(&new_column) {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let lazy = NuLazyFrame::new(true, df.lazy().with_columns(&expressions));
let df = lazy.collect(call.head)?;
Ok(PipelineData::Value(df.into_value(call.head), None))
} else {
let mut other = NuDataFrame::try_from_value(new_column)?.as_series(column_span)?;
let name = match call.get_flag::<String>(engine_state, stack, "name")? {
Some(name) => name,
None => other.name().to_string(),
};
let series = other.rename(&name).clone();
df.as_mut()
.with_column(series)
.map_err(|e| ShellError::GenericError {
error: "Error adding column to dataframe".into(),
msg: e.to_string(),
span: Some(column_span),
help: None,
inner: vec![],
})
.map(|df| {
PipelineData::Value(
NuDataFrame::dataframe_into_value(df.clone(), call.head),
None,
)
})
}
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let lazy: NuLazyFrame = lazy.into_polars().with_columns(&expressions).into();
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(WithColumn {}),
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
])
}
}

86
crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs
View file

@ -1,86 +0,0 @@
use crate::dataframe::values::NuExpression;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ExprAlias;
impl Command for ExprAlias {
fn name(&self) -> &str {
"dfr as"
}
fn usage(&self) -> &str {
"Creates an alias expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"Alias name",
SyntaxShape::String,
"Alias name for the expression",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates and alias expression",
example: "dfr col a | dfr as new_a | dfr into-nu",
result: {
let record = Value::test_record(record! {
"expr" => Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
}),
"alias" => Value::test_string("new_a"),
});
Some(record)
},
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["aka", "abbr", "otherwise"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let alias: String = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr: NuExpression = expr.into_polars().alias(alias.as_str()).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
Box::new(ToNu {}),
])
}
}

78
crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs
View file

@ -1,78 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::arg_where;
#[derive(Clone)]
pub struct ExprArgWhere;
impl Command for ExprArgWhere {
fn name(&self) -> &str {
"dfr arg-where"
}
fn usage(&self) -> &str {
"Creates an expression that returns the arguments where expression is true."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("column name", SyntaxShape::Any, "Expression to evaluate")
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Return a dataframe where the value match the expression",
example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
$df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"b_arg".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["condition", "match", "if"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value: Value = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = arg_where(expr.into_polars()).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::lazy::LazySelect;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprArgWhere {}),
Box::new(ExprAlias {}),
Box::new(LazySelect {}),
])
}
}

68
crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs
View file

@ -1,68 +0,0 @@
use crate::dataframe::values::NuExpression;
use nu_engine::command_prelude::*;
use polars::prelude::col;
#[derive(Clone)]
pub struct ExprCol;
impl Command for ExprCol {
fn name(&self) -> &str {
"dfr col"
}
fn usage(&self) -> &str {
"Creates a named column expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"column name",
SyntaxShape::String,
"Name of column to be used",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a named column expression and converts it to a nu object",
example: "dfr col a | dfr into-nu",
result: Some(Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
})),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["create"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let name: String = call.req(engine_state, stack, 0)?;
let expr: NuExpression = col(name.as_str()).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ExprCol {}), Box::new(ToNu {})])
}
}

108
crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs
View file

@ -1,108 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::concat_str;
#[derive(Clone)]
pub struct ExprConcatStr;
impl Command for ExprConcatStr {
fn name(&self) -> &str {
"dfr concat-str"
}
fn usage(&self) -> &str {
"Creates a concat string expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"separator",
SyntaxShape::String,
"Separator used during the concatenation",
)
.required(
"concat expressions",
SyntaxShape::List(Box::new(SyntaxShape::Any)),
"Expression(s) that define the string concatenation",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a concat string expression",
example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df);
$df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("three")],
),
Column::new(
"b".to_string(),
vec![Value::test_string("two"), Value::test_string("four")],
),
Column::new(
"c".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"concat".to_string(),
vec![
Value::test_string("one-two-2"),
Value::test_string("three-four-4"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["join", "connect", "update"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let separator: String = call.req(engine_state, stack, 0)?;
let value: Value = call.req(engine_state, stack, 1)?;
let expressions = NuExpression::extract_exprs(value)?;
let expr: NuExpression = concat_str(expressions, &separator, false).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::WithColumn;
use crate::dataframe::expressions::alias::ExprAlias;
use crate::dataframe::expressions::col::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprConcatStr {}),
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
Box::new(WithColumn {}),
])
}
}

170
crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs
View file

@ -1,170 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use chrono::{DateTime, FixedOffset};
use nu_engine::command_prelude::*;
use polars::{
datatypes::{DataType, TimeUnit},
prelude::NamedFrom,
series::Series,
};
#[derive(Clone)]
pub struct ExprDatePart;
impl Command for ExprDatePart {
fn name(&self) -> &str {
"dfr datepart"
}
fn usage(&self) -> &str {
"Creates an expression for capturing the specified datepart in a column."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"Datepart name",
SyntaxShape::String,
"Part of the date to capture. Possible values are year, quarter, month, week, weekday, day, hour, minute, second, millisecond, microsecond, nanosecond",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
let dt = DateTime::<FixedOffset>::parse_from_str(
"2021-12-30T01:02:03.123456789 +0000",
"%Y-%m-%dT%H:%M:%S.%9f %z",
)
.expect("date calculation should not fail in test");
vec![
Example {
description: "Creates an expression to capture the year date part",
example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates an expression to capture multiple date parts",
example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" |
dfr with-column [ (dfr col datetime | dfr datepart year | dfr as datetime_year ),
(dfr col datetime | dfr datepart month | dfr as datetime_month ),
(dfr col datetime | dfr datepart day | dfr as datetime_day ),
(dfr col datetime | dfr datepart hour | dfr as datetime_hour ),
(dfr col datetime | dfr datepart minute | dfr as datetime_minute ),
(dfr col datetime | dfr datepart second | dfr as datetime_second ),
(dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#,
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("datetime", &[dt.timestamp_nanos_opt()])
.cast(&DataType::Datetime(TimeUnit::Nanoseconds, None))
.expect("Error casting to datetime type"),
Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64
Series::new("datetime_month", &[12_i8]),
Series::new("datetime_day", &[30_i8]),
Series::new("datetime_hour", &[1_i8]),
Series::new("datetime_minute", &[2_i8]),
Series::new("datetime_second", &[3_i8]),
Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn search_terms(&self) -> Vec<&str> {
vec![
"year",
"month",
"week",
"weekday",
"quarter",
"day",
"hour",
"minute",
"second",
"millisecond",
"microsecond",
"nanosecond",
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let part: Spanned<String> = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr_dt = expr.into_polars().dt();
let expr = match part.item.as_str() {
"year" => expr_dt.year(),
"quarter" => expr_dt.quarter(),
"month" => expr_dt.month(),
"week" => expr_dt.week(),
"day" => expr_dt.day(),
"hour" => expr_dt.hour(),
"minute" => expr_dt.minute(),
"second" => expr_dt.second(),
"millisecond" => expr_dt.millisecond(),
"microsecond" => expr_dt.microsecond(),
"nanosecond" => expr_dt.nanosecond(),
_ => {
return Err(ShellError::UnsupportedInput {
msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item),
input: "value originates from here".to_string(),
msg_span: call.head,
input_span: part.span,
});
}
}.into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
use crate::dataframe::eager::WithColumn;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::expressions::ExprCol;
use crate::dataframe::series::AsDateTime;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprDatePart {}),
Box::new(ExprCol {}),
Box::new(ToNu {}),
Box::new(AsDateTime {}),
Box::new(WithColumn {}),
Box::new(ExprAlias {}),
])
}
}

736
crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs
View file

@ -1,736 +0,0 @@
/// Definition of multiple Expression commands using a macro rule
/// All of these expressions have an identical body and only require
/// to have a change in the name, description and expression function
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
// The structs defined in this file are structs that form part of other commands
// since they share a similar name
macro_rules! expr_command {
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr: NuExpression = expr.into_polars().$func().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new($command {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
])
}
}
};
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr: NuExpression = expr.into_polars().$func($ddof).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new($command {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
])
}
}
};
}
// The structs defined in this file are structs that form part of other commands
// since they share a similar name
macro_rules! lazy_expr_command {
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let lazy = NuLazyFrame::try_from_value(value)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars()
.$func()
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
help: None,
span: None,
inner: vec![],
})?,
);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().$func().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::{
build_test_engine_state, test_dataframe_example,
};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
// the first example should be a for the dataframe case
let example = &$command.examples()[0];
let mut engine_state = build_test_engine_state(vec![Box::new($command {})]);
test_dataframe_example(&mut engine_state, &example)
}
#[test]
fn test_examples_expressions() {
// the second example should be a for the dataframe case
let example = &$command.examples()[1];
let mut engine_state = build_test_engine_state(vec![
Box::new($command {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &example)
}
}
};
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let lazy = NuLazyFrame::try_from_value(value)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars()
.$func($ddof)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
help: None,
span: None,
inner: vec![],
})?,
);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().$func($ddof).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::{
build_test_engine_state, test_dataframe_example,
};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
// the first example should be a for the dataframe case
let example = &$command.examples()[0];
let mut engine_state = build_test_engine_state(vec![Box::new($command {})]);
test_dataframe_example(&mut engine_state, &example)
}
#[test]
fn test_examples_expressions() {
// the second example should be a for the dataframe case
let example = &$command.examples()[1];
let mut engine_state = build_test_engine_state(vec![
Box::new($command {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &example)
}
}
};
}
// ExprList command
// Expands to a command definition for a list expression
expr_command!(
ExprList,
"dfr implode",
"Aggregates a group to a Series.",
vec![Example {
description: "",
example: "",
result: None,
}],
implode,
test_implode
);
// ExprAggGroups command
// Expands to a command definition for a agg groups expression
expr_command!(
ExprAggGroups,
"dfr agg-groups",
"Creates an agg_groups expression.",
vec![Example {
description: "",
example: "",
result: None,
}],
agg_groups,
test_groups
);
// ExprCount command
// Expands to a command definition for a count expression
expr_command!(
ExprCount,
"dfr count",
"Creates a count expression.",
vec![Example {
description: "",
example: "",
result: None,
}],
count,
test_count
);
// ExprNot command
// Expands to a command definition for a not expression
expr_command!(
ExprNot,
"dfr expr-not",
"Creates a not expression.",
vec![Example {
description: "Creates a not expression",
example: "(dfr col a) > 2) | dfr expr-not",
result: None,
},],
not,
test_not
);
// ExprMax command
// Expands to a command definition for max aggregation
lazy_expr_command!(
ExprMax,
"dfr max",
"Creates a max expression or aggregates columns to their max value.",
vec![
Example {
description: "Max value from columns in a dataframe",
example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(6)],),
Column::new("b".to_string(), vec![Value::test_int(4)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Max aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr max)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_int(4), Value::test_int(1)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
max,
test_max
);
// ExprMin command
// Expands to a command definition for min aggregation
lazy_expr_command!(
ExprMin,
"dfr min",
"Creates a min expression or aggregates columns to their min value.",
vec![
Example {
description: "Min value from columns in a dataframe",
example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)],),
Column::new("b".to_string(), vec![Value::test_int(1)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Min aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr min)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(1)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
min,
test_min
);
// ExprSum command
// Expands to a command definition for sum aggregation
lazy_expr_command!(
ExprSum,
"dfr sum",
"Creates a sum expression for an aggregation or aggregates columns to their sum value.",
vec![
Example {
description: "Sums all columns in a dataframe",
example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(11)],),
Column::new("b".to_string(), vec![Value::test_int(7)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Sum aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr sum)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_int(6), Value::test_int(1)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
sum,
test_sum
);
// ExprMean command
// Expands to a command definition for mean aggregation
lazy_expr_command!(
ExprMean,
"dfr mean",
"Creates a mean expression for an aggregation or aggregates columns to their mean value.",
vec![
Example {
description: "Mean value from columns in a dataframe",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_float(4.0)],),
Column::new("b".to_string(), vec![Value::test_float(2.0)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Mean aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr mean)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_float(3.0), Value::test_float(1.0)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
mean,
test_mean
);
// ExprMedian command
// Expands to a command definition for median aggregation
expr_command!(
ExprMedian,
"dfr median",
"Creates a median expression for an aggregation.",
vec![Example {
description: "Median aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr median)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_float(3.0), Value::test_float(1.0)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},],
median,
test_median
);
// ExprStd command
// Expands to a command definition for std aggregation
lazy_expr_command!(
ExprStd,
"dfr std",
"Creates a std expression for an aggregation of std value from columns in a dataframe.",
vec![
Example {
description: "Std value from columns in a dataframe",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_float(2.0)],),
Column::new("b".to_string(), vec![Value::test_float(0.0)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Std aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr std)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_float(0.0), Value::test_float(0.0)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
std,
test_std,
1
);
// ExprVar command
// Expands to a command definition for var aggregation
lazy_expr_command!(
ExprVar,
"dfr var",
"Create a var expression for an aggregation.",
vec![
Example {
description:
"Var value from columns in a dataframe or aggregates columns to their var value",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_float(4.0)],),
Column::new("b".to_string(), vec![Value::test_float(0.0)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Var aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr var)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_float(0.0), Value::test_float(0.0)],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
],
var,
test_var,
1
);

116
crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs
View file

@ -1,116 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::{lit, DataType};
#[derive(Clone)]
pub struct ExprIsIn;
impl Command for ExprIsIn {
fn name(&self) -> &str {
"dfr is-in"
}
fn usage(&self) -> &str {
"Creates an is-in expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"list",
SyntaxShape::List(Box::new(SyntaxShape::Any)),
"List to check if values are in",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a is-in expression",
example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
$df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_string("one"),
Value::test_string("two"),
Value::test_string("three"),
],
),
Column::new(
"b".to_string(),
vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
),
Column::new(
"a_in".to_string(),
vec![
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["check", "contained", "is-contain", "match"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let list: Vec<Value> = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let values =
NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?;
let list = values.as_series(call.head)?;
if matches!(list.dtype(), DataType::Object(..)) {
return Err(ShellError::IncompatibleParametersSingle {
msg: "Cannot use a mixed list as argument".into(),
span: call.head,
});
}
let expr: NuExpression = expr.into_polars().is_in(lit(list)).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::WithColumn;
use crate::dataframe::expressions::alias::ExprAlias;
use crate::dataframe::expressions::col::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprIsIn {}),
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
Box::new(WithColumn {}),
])
}
}

69
crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs
View file

@ -1,69 +0,0 @@
use crate::dataframe::values::NuExpression;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ExprLit;
impl Command for ExprLit {
fn name(&self) -> &str {
"dfr lit"
}
fn usage(&self) -> &str {
"Creates a literal expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"literal",
SyntaxShape::Any,
"literal to construct the expression",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Created a literal expression and converts it to a nu object",
example: "dfr lit 2 | dfr into-nu",
result: Some(Value::test_record(record! {
"expr" => Value::test_string("literal"),
"value" => Value::test_string("2"),
})),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["string", "literal", "expression"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let literal: Value = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(literal)?;
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ExprLit {}), Box::new(ToNu {})])
}
}

62
crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs
View file

@ -1,62 +0,0 @@
mod alias;
mod arg_where;
mod col;
mod concat_str;
mod datepart;
mod expressions_macro;
mod is_in;
mod lit;
mod otherwise;
mod quantile;
mod when;
use nu_protocol::engine::StateWorkingSet;
pub(crate) use crate::dataframe::expressions::alias::ExprAlias;
use crate::dataframe::expressions::arg_where::ExprArgWhere;
pub(super) use crate::dataframe::expressions::col::ExprCol;
pub(super) use crate::dataframe::expressions::concat_str::ExprConcatStr;
pub(crate) use crate::dataframe::expressions::datepart::ExprDatePart;
pub(crate) use crate::dataframe::expressions::expressions_macro::*;
pub(super) use crate::dataframe::expressions::is_in::ExprIsIn;
pub(super) use crate::dataframe::expressions::lit::ExprLit;
pub(super) use crate::dataframe::expressions::otherwise::ExprOtherwise;
pub(super) use crate::dataframe::expressions::quantile::ExprQuantile;
pub(super) use crate::dataframe::expressions::when::ExprWhen;
pub fn add_expressions(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
( $( $command:expr ),* ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
// Dataframe commands
bind_command!(
ExprAlias,
ExprArgWhere,
ExprCol,
ExprConcatStr,
ExprCount,
ExprLit,
ExprWhen,
ExprOtherwise,
ExprQuantile,
ExprList,
ExprAggGroups,
ExprCount,
ExprIsIn,
ExprNot,
ExprMax,
ExprMin,
ExprSum,
ExprMean,
ExprMedian,
ExprStd,
ExprVar,
ExprDatePart
);
}

126
crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs
View file

@ -1,126 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ExprOtherwise;
impl Command for ExprOtherwise {
fn name(&self) -> &str {
"dfr otherwise"
}
fn usage(&self) -> &str {
"Completes a when expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"otherwise expression",
SyntaxShape::Any,
"expression to apply when no when predicate matches",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create a when conditions",
example: "dfr when ((dfr col a) > 2) 4 | dfr otherwise 5",
result: None,
},
Example {
description: "Create a when conditions",
example:
"dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6 | dfr otherwise 0",
result: None,
},
Example {
description: "Create a new column for the dataframe",
example: r#"[[a b]; [6 2] [1 4] [4 1]]
| dfr into-lazy
| dfr with-column (
dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c
)
| dfr with-column (
dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d
)
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
),
Column::new(
"d".to_string(),
vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn search_terms(&self) -> Vec<&str> {
vec!["condition", "else"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let otherwise_predicate: Value = call.req(engine_state, stack, 0)?;
let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?;
let value = input.into_value(call.head)?;
let complete: NuExpression = match NuWhen::try_from_value(value)? {
NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(),
NuWhen::ChainedThen(chained_when) => chained_when
.otherwise(otherwise_predicate.into_polars())
.into(),
};
Ok(PipelineData::Value(complete.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use crate::dataframe::eager::{ToNu, WithColumn};
use crate::dataframe::expressions::when::ExprWhen;
use crate::dataframe::expressions::{ExprAlias, ExprCol};
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(WithColumn {}),
Box::new(ExprCol {}),
Box::new(ExprAlias {}),
Box::new(ExprWhen {}),
Box::new(ExprOtherwise {}),
Box::new(ToNu {}),
])
}
}

101
crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs
View file

@ -1,101 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::{lit, QuantileInterpolOptions};
#[derive(Clone)]
pub struct ExprQuantile;
impl Command for ExprQuantile {
fn name(&self) -> &str {
"dfr quantile"
}
fn usage(&self) -> &str {
"Aggregates the columns to the selected quantile."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"quantile",
SyntaxShape::Number,
"quantile value for quantile operation",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Quantile aggregation for a group-by",
example: r#"[[a b]; [one 2] [one 4] [two 1]]
| dfr into-df
| dfr group-by a
| dfr agg (dfr col b | dfr quantile 0.5)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("two")],
),
Column::new(
"b".to_string(),
vec![Value::test_float(4.0), Value::test_float(1.0)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["statistics", "percentile", "distribution"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
let quantile: f64 = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr
.into_polars()
.quantile(lit(quantile), QuantileInterpolOptions::default())
.into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprQuantile {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
])
}
}

147
crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs
View file

@ -1,147 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen};
use nu_engine::command_prelude::*;
use polars::prelude::when;
#[derive(Clone)]
pub struct ExprWhen;
impl Command for ExprWhen {
fn name(&self) -> &str {
"dfr when"
}
fn usage(&self) -> &str {
"Creates and modifies a when expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"when expression",
SyntaxShape::Any,
"when expression used for matching",
)
.required(
"then expression",
SyntaxShape::Any,
"expression that will be applied when predicate is true",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create a when conditions",
example: "dfr when ((dfr col a) > 2) 4",
result: None,
},
Example {
description: "Create a when conditions",
example: "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6",
result: None,
},
Example {
description: "Create a new column for the dataframe",
example: r#"[[a b]; [6 2] [1 4] [4 1]]
| dfr into-lazy
| dfr with-column (
dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c
)
| dfr with-column (
dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d
)
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
),
Column::new(
"d".to_string(),
vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn search_terms(&self) -> Vec<&str> {
vec!["condition", "match", "if", "else"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let when_predicate: Value = call.req(engine_state, stack, 0)?;
let when_predicate = NuExpression::try_from_value(when_predicate)?;
let then_predicate: Value = call.req(engine_state, stack, 1)?;
let then_predicate = NuExpression::try_from_value(then_predicate)?;
let value = input.into_value(call.head)?;
let when_then: NuWhen = match value {
Value::Nothing { .. } => when(when_predicate.into_polars())
.then(then_predicate.into_polars())
.into(),
v => match NuWhen::try_from_value(v)? {
NuWhen::Then(when_then) => when_then
.when(when_predicate.into_polars())
.then(then_predicate.into_polars())
.into(),
NuWhen::ChainedThen(when_then_then) => when_then_then
.when(when_predicate.into_polars())
.then(then_predicate.into_polars())
.into(),
},
};
Ok(PipelineData::Value(when_then.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use crate::dataframe::eager::{ToNu, WithColumn};
use crate::dataframe::expressions::otherwise::ExprOtherwise;
use crate::dataframe::expressions::{ExprAlias, ExprCol};
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(WithColumn {}),
Box::new(ExprCol {}),
Box::new(ExprAlias {}),
Box::new(ExprWhen {}),
Box::new(ExprOtherwise {}),
Box::new(ToNu {}),
])
}
}

216
crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs
View file

@ -1,216 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy};
use nu_engine::command_prelude::*;
use polars::{datatypes::DataType, prelude::Expr};
#[derive(Clone)]
pub struct LazyAggregate;
impl Command for LazyAggregate {
fn name(&self) -> &str {
"dfr agg"
}
fn usage(&self) -> &str {
"Performs a series of aggregations from a group-by."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"Group-by expressions",
SyntaxShape::Any,
"Expression(s) that define the aggregations to be applied",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Group by and perform an aggregation",
example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
| dfr into-df
| dfr group-by a
| dfr agg [
(dfr col b | dfr min | dfr as "b_min")
(dfr col b | dfr max | dfr as "b_max")
(dfr col b | dfr sum | dfr as "b_sum")
]"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"b_min".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"b_max".to_string(),
vec![Value::test_int(4), Value::test_int(6)],
),
Column::new(
"b_sum".to_string(),
vec![Value::test_int(6), Value::test_int(10)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Group by and perform an aggregation",
example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
| dfr into-lazy
| dfr group-by a
| dfr agg [
(dfr col b | dfr min | dfr as "b_min")
(dfr col b | dfr max | dfr as "b_max")
(dfr col b | dfr sum | dfr as "b_sum")
]
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"b_min".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"b_max".to_string(),
vec![Value::test_int(4), Value::test_int(6)],
),
Column::new(
"b_sum".to_string(),
vec![Value::test_int(6), Value::test_int(10)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let group_by = NuLazyGroupBy::try_from_pipeline(input, call.head)?;
if let Some(schema) = &group_by.schema {
for expr in expressions.iter() {
if let Some(name) = get_col_name(expr) {
let dtype = schema.get(name.as_str());
if matches!(dtype, Some(DataType::Object(..))) {
return Err(ShellError::GenericError {
error: "Object type column not supported for aggregation".into(),
msg: format!("Column '{name}' is type Object"),
span: Some(call.head),
help: Some("Aggregations cannot be performed on Object type columns. Use dtype command to check column types".into()),
inner: vec![],
});
}
}
}
}
let lazy = NuLazyFrame {
from_eager: group_by.from_eager,
lazy: Some(group_by.into_polars().agg(&expressions)),
schema: None,
};
let res = lazy.into_value(call.head)?;
Ok(PipelineData::Value(res, None))
}
}
fn get_col_name(expr: &Expr) -> Option<String> {
match expr {
Expr::Column(column) => Some(column.to_string()),
Expr::Agg(agg) => match agg {
polars::prelude::AggExpr::Min { input: e, .. }
| polars::prelude::AggExpr::Max { input: e, .. }
| polars::prelude::AggExpr::Median(e)
| polars::prelude::AggExpr::NUnique(e)
| polars::prelude::AggExpr::First(e)
| polars::prelude::AggExpr::Last(e)
| polars::prelude::AggExpr::Mean(e)
| polars::prelude::AggExpr::Implode(e)
| polars::prelude::AggExpr::Count(e, _)
| polars::prelude::AggExpr::Sum(e)
| polars::prelude::AggExpr::AggGroups(e)
| polars::prelude::AggExpr::Std(e, _)
| polars::prelude::AggExpr::Var(e, _) => get_col_name(e.as_ref()),
polars::prelude::AggExpr::Quantile { expr, .. } => get_col_name(expr.as_ref()),
},
Expr::Filter { input: expr, .. }
| Expr::Slice { input: expr, .. }
| Expr::Cast { expr, .. }
| Expr::Sort { expr, .. }
| Expr::Gather { expr, .. }
| Expr::SortBy { expr, .. }
| Expr::Exclude(expr, _)
| Expr::Alias(expr, _)
| Expr::KeepName(expr)
| Expr::Explode(expr) => get_col_name(expr.as_ref()),
Expr::Ternary { .. }
| Expr::AnonymousFunction { .. }
| Expr::Function { .. }
| Expr::Columns(_)
| Expr::DtypeColumn(_)
| Expr::Literal(_)
| Expr::BinaryExpr { .. }
| Expr::Window { .. }
| Expr::Wildcard
| Expr::RenameAlias { .. }
| Expr::Len
| Expr::Nth(_)
| Expr::SubPlan(_, _)
| Expr::Selector(_) => None,
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum};
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
Box::new(ExprAlias {}),
Box::new(ExprMin {}),
Box::new(ExprMax {}),
Box::new(ExprSum {}),
])
}
}

73
crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs
View file

@ -1,73 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyCollect;
impl Command for LazyCollect {
fn name(&self) -> &str {
"dfr collect"
}
fn usage(&self) -> &str {
"Collect lazy dataframe into eager dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop duplicates",
example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let eager = lazy.collect(call.head)?;
let value = Value::custom(Box::new(eager), call.head);
Ok(PipelineData::Value(value, None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyCollect {})])
}
}

153
crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs
View file

@ -1,153 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyExplode;
impl Command for LazyExplode {
fn name(&self) -> &str {
"dfr explode"
}
fn usage(&self) -> &str {
"Explodes a dataframe or creates a explode expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"columns",
SyntaxShape::String,
"columns to explode, only applicable for dataframes",
)
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Explode the specified dataframe",
example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr explode hobbies | dfr collect",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"id".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(2),
Value::test_int(2),
]),
Column::new(
"name".to_string(),
vec![
Value::test_string("Mercy"),
Value::test_string("Mercy"),
Value::test_string("Bob"),
Value::test_string("Bob"),
]),
Column::new(
"hobbies".to_string(),
vec![
Value::test_string("Cycling"),
Value::test_string("Knitting"),
Value::test_string("Skiing"),
Value::test_string("Football"),
]),
], None).expect("simple df for test should not fail")
.into_value(Span::test_data()),
)
},
Example {
description: "Select a column and explode the values",
example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr explode)",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"hobbies".to_string(),
vec![
Value::test_string("Cycling"),
Value::test_string("Knitting"),
Value::test_string("Skiing"),
Value::test_string("Football"),
]),
], None).expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
explode(call, input)
}
}
pub(crate) fn explode(call: &Call, input: PipelineData) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
let columns: Vec<String> = call
.positional_iter()
.filter_map(|e| e.as_string())
.collect();
let exploded = df
.into_polars()
.explode(columns.iter().map(AsRef::as_ref).collect::<Vec<&str>>());
Ok(PipelineData::Value(
NuLazyFrame::from(exploded).into_value(call.head)?,
None,
))
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().explode().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(LazyExplode {})]);
test_dataframe_example(&mut engine_state, &LazyExplode.examples()[0]);
}
#[ignore]
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(LazyExplode {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &LazyExplode.examples()[1]);
}
}

92
crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs
View file

@ -1,92 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyFetch;
impl Command for LazyFetch {
fn name(&self) -> &str {
"dfr fetch"
}
fn usage(&self) -> &str {
"Collects the lazyframe to the selected rows."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"rows",
SyntaxShape::Int,
"number of rows to be fetched from lazyframe",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Fetch a rows from the dataframe",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(6), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let rows: i64 = call.req(engine_state, stack, 0)?;
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let eager: NuDataFrame = lazy
.into_polars()
.fetch(rows as usize)
.map_err(|e| ShellError::GenericError {
error: "Error fetching rows".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(PipelineData::Value(
NuDataFrame::into_value(eager, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyFetch {})])
}
}

143
crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs
View file

@ -1,143 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyFillNA;
impl Command for LazyFillNA {
fn name(&self) -> &str {
"dfr fill-nan"
}
fn usage(&self) -> &str {
"Replaces NaN values with the given expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"fill",
SyntaxShape::Any,
"Expression to use to fill the NAN values",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Fills the NaN values with 0",
example: "[1 2 NaN 3 NaN] | dfr into-df | dfr fill-nan 0",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(0),
Value::test_int(3),
Value::test_int(0),
],
)],
None,
)
.expect("Df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Fills the NaN values of a whole dataframe",
example: "[[a b]; [0.2 1] [0.1 NaN]] | dfr into-df | dfr fill-nan 0",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_float(0.2), Value::test_float(0.1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(1), Value::test_int(0)],
),
],
None,
)
.expect("Df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let fill: Value = call.req(engine_state, stack, 0)?;
let value = input.into_value(call.head)?;
if NuExpression::can_downcast(&value) {
let expr = NuExpression::try_from_value(value)?;
let fill = NuExpression::try_from_value(fill)?.into_polars();
let expr: NuExpression = expr.into_polars().fill_nan(fill).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
} else {
let val_span = value.span();
let frame = NuDataFrame::try_from_value(value)?;
let columns = frame.columns(val_span)?;
let dataframe = columns
.into_iter()
.map(|column| {
let column_name = column.name().to_string();
let values = column
.into_iter()
.map(|value| {
let span = value.span();
match value {
Value::Float { val, .. } => {
if val.is_nan() {
fill.clone()
} else {
value
}
}
Value::List { vals, .. } => {
NuDataFrame::fill_list_nan(vals, span, fill.clone())
}
_ => value,
}
})
.collect::<Vec<Value>>();
Column::new(column_name, values)
})
.collect::<Vec<Column>>();
Ok(PipelineData::Value(
NuDataFrame::try_from_columns(dataframe, None)?.into_value(call.head),
None,
))
}
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyFillNA {})])
}
}

93
crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs
View file

@ -1,93 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyFillNull;
impl Command for LazyFillNull {
fn name(&self) -> &str {
"dfr fill-null"
}
fn usage(&self) -> &str {
"Replaces NULL values with the given expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"fill",
SyntaxShape::Any,
"Expression to use to fill the null values",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Fills the null values by 0",
example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr fill-null 0",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_int(0),
Value::test_int(0),
Value::test_int(1),
Value::test_int(2),
Value::test_int(2),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let fill: Value = call.req(engine_state, stack, 0)?;
let value = input.into_value(call.head)?;
if NuExpression::can_downcast(&value) {
let expr = NuExpression::try_from_value(value)?;
let fill = NuExpression::try_from_value(fill)?.into_polars();
let expr: NuExpression = expr.into_polars().fill_null(fill).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
} else {
let lazy = NuLazyFrame::try_from_value(value)?;
let expr = NuExpression::try_from_value(fill)?.into_polars();
let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().fill_null(expr));
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
}
#[cfg(test)]
mod test {
use super::super::super::series::Shift;
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyFillNull {}), Box::new(Shift {})])
}
}

83
crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs
View file

@ -1,83 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyFilter;
impl Command for LazyFilter {
fn name(&self) -> &str {
"dfr filter"
}
fn usage(&self) -> &str {
"Filter dataframe based in expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"filter expression",
SyntaxShape::Any,
"Expression that define the column selection",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Filter dataframe using an expression",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr filter ((dfr col a) >= 4)",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(6), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value: Value = call.req(engine_state, stack, 0)?;
let expression = NuExpression::try_from_value(value)?;
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars().filter(expression.into_polars()),
);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyFilter {})])
}
}

126
crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs
View file

@ -1,126 +0,0 @@
use super::explode::explode;
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazyFlatten;
impl Command for LazyFlatten {
fn name(&self) -> &str {
"dfr flatten"
}
fn usage(&self) -> &str {
"An alias for dfr explode."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"columns",
SyntaxShape::String,
"columns to flatten, only applicable for dataframes",
)
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Flatten the specified dataframe",
example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr flatten hobbies | dfr collect",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"id".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(2),
Value::test_int(2),
]),
Column::new(
"name".to_string(),
vec![
Value::test_string("Mercy"),
Value::test_string("Mercy"),
Value::test_string("Bob"),
Value::test_string("Bob"),
]),
Column::new(
"hobbies".to_string(),
vec![
Value::test_string("Cycling"),
Value::test_string("Knitting"),
Value::test_string("Skiing"),
Value::test_string("Football"),
]),
], None).expect("simple df for test should not fail")
.into_value(Span::test_data()),
)
},
Example {
description: "Select a column and flatten the values",
example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr flatten)",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"hobbies".to_string(),
vec![
Value::test_string("Cycling"),
Value::test_string("Knitting"),
Value::test_string("Skiing"),
Value::test_string("Football"),
]),
], None).expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
explode(call, input)
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(LazyFlatten {})]);
test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[0]);
}
#[ignore]
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(LazyFlatten {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[1]);
}
}

161
crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs
View file

@ -1,161 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy};
use nu_engine::command_prelude::*;
use polars::prelude::Expr;
#[derive(Clone)]
pub struct ToLazyGroupBy;
impl Command for ToLazyGroupBy {
fn name(&self) -> &str {
"dfr group-by"
}
fn usage(&self) -> &str {
"Creates a group-by object that can be used for other aggregations."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"Group-by expressions",
SyntaxShape::Any,
"Expression(s) that define the lazy group-by",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Group by and perform an aggregation",
example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
| dfr into-df
| dfr group-by a
| dfr agg [
(dfr col b | dfr min | dfr as "b_min")
(dfr col b | dfr max | dfr as "b_max")
(dfr col b | dfr sum | dfr as "b_sum")
]"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"b_min".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"b_max".to_string(),
vec![Value::test_int(4), Value::test_int(6)],
),
Column::new(
"b_sum".to_string(),
vec![Value::test_int(6), Value::test_int(10)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Group by and perform an aggregation",
example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
| dfr into-lazy
| dfr group-by a
| dfr agg [
(dfr col b | dfr min | dfr as "b_min")
(dfr col b | dfr max | dfr as "b_max")
(dfr col b | dfr sum | dfr as "b_sum")
]
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"b_min".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"b_max".to_string(),
vec![Value::test_int(4), Value::test_int(6)],
),
Column::new(
"b_sum".to_string(),
vec![Value::test_int(6), Value::test_int(10)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
if expressions
.iter()
.any(|expr| !matches!(expr, Expr::Column(..)))
{
let value: Value = call.req(engine_state, stack, 0)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "Expected only Col expressions".into(),
span: value.span(),
});
}
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let group_by = NuLazyGroupBy {
schema: lazy.schema.clone(),
from_eager: lazy.from_eager,
group_by: Some(lazy.into_polars().group_by(&expressions)),
};
Ok(PipelineData::Value(group_by.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum};
use crate::dataframe::lazy::aggregate::LazyAggregate;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
Box::new(ExprAlias {}),
Box::new(ExprMin {}),
Box::new(ExprMax {}),
Box::new(ExprSum {}),
])
}
}

252
crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs
View file

@ -1,252 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{Expr, JoinType};
#[derive(Clone)]
pub struct LazyJoin;
impl Command for LazyJoin {
fn name(&self) -> &str {
"dfr join"
}
fn usage(&self) -> &str {
"Joins a lazy frame with other lazy frame."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("other", SyntaxShape::Any, "LazyFrame to join with")
.required("left_on", SyntaxShape::Any, "Left column(s) to join on")
.required("right_on", SyntaxShape::Any, "Right column(s) to join on")
.switch(
"inner",
"inner join between lazyframes (default)",
Some('i'),
)
.switch("left", "left join between lazyframes", Some('l'))
.switch("outer", "outer join between lazyframes", Some('o'))
.switch("cross", "cross join between lazyframes", Some('c'))
.named(
"suffix",
SyntaxShape::String,
"Suffix to use on columns with same name",
Some('s'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Join two lazy dataframes",
example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-lazy);
let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
$df_a | dfr join $df_b a foo | dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(1),
Value::test_int(1),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
Value::test_string("c"),
],
),
Column::new(
"c".to_string(),
vec![
Value::test_int(0),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
],
),
Column::new(
"bar".to_string(),
vec![
Value::test_string("a"),
Value::test_string("c"),
Value::test_string("a"),
Value::test_string("a"),
],
),
Column::new(
"ham".to_string(),
vec![
Value::test_string("let"),
Value::test_string("var"),
Value::test_string("let"),
Value::test_string("let"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Join one eager dataframe with a lazy dataframe",
example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-df);
let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
$df_a | dfr join $df_b a foo"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(1),
Value::test_int(1),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
Value::test_string("c"),
],
),
Column::new(
"c".to_string(),
vec![
Value::test_int(0),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
],
),
Column::new(
"bar".to_string(),
vec![
Value::test_string("a"),
Value::test_string("c"),
Value::test_string("a"),
Value::test_string("a"),
],
),
Column::new(
"ham".to_string(),
vec![
Value::test_string("let"),
Value::test_string("var"),
Value::test_string("let"),
Value::test_string("let"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let left = call.has_flag(engine_state, stack, "left")?;
let outer = call.has_flag(engine_state, stack, "outer")?;
let cross = call.has_flag(engine_state, stack, "cross")?;
let how = if left {
JoinType::Left
} else if outer {
JoinType::Outer { coalesce: true }
} else if cross {
JoinType::Cross
} else {
JoinType::Inner
};
let other: Value = call.req(engine_state, stack, 0)?;
let other = NuLazyFrame::try_from_value(other)?;
let other = other.into_polars();
let left_on: Value = call.req(engine_state, stack, 1)?;
let left_on = NuExpression::extract_exprs(left_on)?;
let right_on: Value = call.req(engine_state, stack, 2)?;
let right_on = NuExpression::extract_exprs(right_on)?;
if left_on.len() != right_on.len() {
let right_on: Value = call.req(engine_state, stack, 2)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "The right column list has a different size to the left column list".into(),
span: right_on.span(),
});
}
// Checking that both list of expressions are made out of col expressions or strings
for (index, list) in &[(1usize, &left_on), (2, &left_on)] {
if list.iter().any(|expr| !matches!(expr, Expr::Column(..))) {
let value: Value = call.req(engine_state, stack, *index)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "Expected only a string, col expressions or list of strings".into(),
span: value.span(),
});
}
}
let suffix: Option<String> = call.get_flag(engine_state, stack, "suffix")?;
let suffix = suffix.unwrap_or_else(|| "_x".into());
let value = input.into_value(call.head)?;
let lazy = NuLazyFrame::try_from_value(value)?;
let from_eager = lazy.from_eager;
let lazy = lazy.into_polars();
let lazy = lazy
.join_builder()
.with(other)
.left_on(left_on)
.right_on(right_on)
.how(how)
.force_parallel(true)
.suffix(suffix)
.finish();
let lazy = NuLazyFrame::new(from_eager, lazy);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyJoin {})])
}
}

246
crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs
View file

@ -1,246 +0,0 @@
/// Definition of multiple lazyframe commands using a macro rule
/// All of these commands have an identical body and only require
/// to have a change in the name, description and function
use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
use nu_engine::command_prelude::*;
macro_rules! lazy_command {
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func());
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new($command {})])
}
}
};
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddot: expr) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddot));
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new($command {})])
}
}
};
($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident?, $test: ident) => {
#[derive(Clone)]
pub struct $command;
impl Command for $command {
fn name(&self) -> &str {
$name
}
fn usage(&self) -> &str {
$desc
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
$examples
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars()
.$func()
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
help: None,
span: None,
inner: vec![],
})?,
);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod $test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new($command {})])
}
}
};
}
// LazyReverse command
// Expands to a command definition for reverse
lazy_command!(
LazyReverse,
"dfr reverse",
"Reverses the LazyFrame",
vec![Example {
description: "Reverses the dataframe.",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),],
),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},],
reverse,
test_reverse
);
// LazyCache command
// Expands to a command definition for cache
lazy_command!(
LazyCache,
"dfr cache",
"Caches operations in a new LazyFrame.",
vec![Example {
description: "Caches the result into a new LazyFrame",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse | dfr cache",
result: None,
}],
cache,
test_cache
);
// LazyMedian command
// Expands to a command definition for median aggregation
lazy_command!(
LazyMedian,
"dfr median",
"Aggregates columns to their median value",
vec![Example {
description: "Median value from columns in a dataframe",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr median",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_float(4.0)],),
Column::new("b".to_string(), vec![Value::test_float(2.0)],),
],
None
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},],
median?,
test_median
);

65
crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs
View file

@ -1,65 +0,0 @@
pub mod aggregate;
mod collect;
mod explode;
mod fetch;
mod fill_nan;
mod fill_null;
mod filter;
mod flatten;
pub mod groupby;
mod join;
mod macro_commands;
mod quantile;
mod select;
mod sort_by_expr;
mod to_lazy;
use nu_protocol::engine::StateWorkingSet;
use crate::dataframe::lazy::aggregate::LazyAggregate;
pub use crate::dataframe::lazy::collect::LazyCollect;
use crate::dataframe::lazy::fetch::LazyFetch;
use crate::dataframe::lazy::fill_nan::LazyFillNA;
pub use crate::dataframe::lazy::fill_null::LazyFillNull;
use crate::dataframe::lazy::filter::LazyFilter;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
use crate::dataframe::lazy::join::LazyJoin;
pub(crate) use crate::dataframe::lazy::macro_commands::*;
use crate::dataframe::lazy::quantile::LazyQuantile;
pub(crate) use crate::dataframe::lazy::select::LazySelect;
use crate::dataframe::lazy::sort_by_expr::LazySortBy;
pub use crate::dataframe::lazy::to_lazy::ToLazyFrame;
pub use explode::LazyExplode;
pub use flatten::LazyFlatten;
pub fn add_lazy_decls(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
( $( $command:expr ),* ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
// Dataframe commands
bind_command!(
LazyAggregate,
LazyCache,
LazyCollect,
LazyFetch,
LazyFillNA,
LazyFillNull,
LazyFilter,
LazyJoin,
LazyQuantile,
LazyMedian,
LazyReverse,
LazySelect,
LazySortBy,
ToLazyFrame,
ToLazyGroupBy,
LazyExplode,
LazyFlatten
);
}

87
crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs
View file

@ -1,87 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{lit, QuantileInterpolOptions};
#[derive(Clone)]
pub struct LazyQuantile;
impl Command for LazyQuantile {
fn name(&self) -> &str {
"dfr quantile"
}
fn usage(&self) -> &str {
"Aggregates the columns to the selected quantile."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"quantile",
SyntaxShape::Number,
"quantile value for quantile operation",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "quantile value from columns in a dataframe",
example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr quantile 0.5",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_float(4.0)]),
Column::new("b".to_string(), vec![Value::test_float(2.0)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
let quantile: f64 = call.req(engine_state, stack, 0)?;
let lazy = NuLazyFrame::try_from_value(value)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars()
.quantile(lit(quantile), QuantileInterpolOptions::default())
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
help: None,
span: None,
inner: vec![],
})?,
);
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazyQuantile {})])
}
}

75
crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs
View file

@ -1,75 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LazySelect;
impl Command for LazySelect {
fn name(&self) -> &str {
"dfr select"
}
fn usage(&self) -> &str {
"Selects columns from lazyframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"select expressions",
SyntaxShape::Any,
"Expression(s) that define the column selection",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Select a column from the dataframe",
example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr select a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(6), Value::test_int(4), Value::test_int(2)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().select(&expressions));
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazySelect {})])
}
}

159
crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs
View file

@ -1,159 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::chunked_array::ops::SortMultipleOptions;
#[derive(Clone)]
pub struct LazySortBy;
impl Command for LazySortBy {
fn name(&self) -> &str {
"dfr sort-by"
}
fn usage(&self) -> &str {
"Sorts a lazy dataframe based on expression(s)."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest(
"sort expression",
SyntaxShape::Any,
"sort expression for the dataframe",
)
.named(
"reverse",
SyntaxShape::List(Box::new(SyntaxShape::Boolean)),
"Reverse sorting. Default is false",
Some('r'),
)
.switch(
"nulls-last",
"nulls are shown last in the dataframe",
Some('n'),
)
.switch("maintain-order", "Maintains order during sort", Some('m'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Sort dataframe by one column",
example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sort-by a",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(4), Value::test_int(6)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(4), Value::test_int(1), Value::test_int(2)],
),
], None)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Sort column using two columns",
example:
"[[a b]; [6 2] [1 1] [1 4] [2 4]] | dfr into-df | dfr sort-by [a b] -r [false true]",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(2),
Value::test_int(6),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_int(4),
Value::test_int(1),
Value::test_int(4),
Value::test_int(2),
],
),
], None)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let nulls_last = call.has_flag(engine_state, stack, "nulls-last")?;
let maintain_order = call.has_flag(engine_state, stack, "maintain-order")?;
let reverse: Option<Vec<bool>> = call.get_flag(engine_state, stack, "reverse")?;
let reverse = match reverse {
Some(list) => {
if expressions.len() != list.len() {
let span = call
.get_flag::<Value>(engine_state, stack, "reverse")?
.expect("already checked and it exists")
.span();
return Err(ShellError::GenericError {
error: "Incorrect list size".into(),
msg: "Size doesn't match expression list".into(),
span: Some(span),
help: None,
inner: vec![],
});
} else {
list
}
}
None => expressions.iter().map(|_| false).collect::<Vec<bool>>(),
};
let sort_options = SortMultipleOptions {
descending: reverse,
nulls_last,
multithreaded: true,
maintain_order,
};
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars().sort_by_exprs(&expressions, sort_options),
);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(LazySortBy {})])
}
}

53
crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs
View file

@ -1,53 +0,0 @@
use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ToLazyFrame;
impl Command for ToLazyFrame {
fn name(&self) -> &str {
"dfr into-lazy"
}
fn usage(&self) -> &str {
"Converts a dataframe into a lazy dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s'),
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Takes a dictionary and creates a lazy dataframe",
example: "[[a b];[1 2] [3 4]] | dfr into-lazy",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema)?;
let lazy = NuLazyFrame::from_dataframe(df);
let value = Value::custom(Box::new(lazy), call.head);
Ok(PipelineData::Value(value, None))
}
}

36
crates/nu-cmd-dataframe/src/dataframe/mod.rs
View file

@ -1,36 +0,0 @@
mod eager;
mod expressions;
mod lazy;
mod series;
mod stub;
mod utils;
mod values;
pub use eager::add_eager_decls;
pub use expressions::add_expressions;
pub use lazy::add_lazy_decls;
pub use series::add_series_decls;
use nu_protocol::engine::{EngineState, StateWorkingSet};
pub fn add_dataframe_context(mut engine_state: EngineState) -> EngineState {
let delta = {
let mut working_set = StateWorkingSet::new(&engine_state);
working_set.add_decl(Box::new(stub::Dfr));
add_series_decls(&mut working_set);
add_eager_decls(&mut working_set);
add_expressions(&mut working_set);
add_lazy_decls(&mut working_set);
working_set.render()
};
if let Err(err) = engine_state.merge_delta(delta) {
eprintln!("Error creating dataframe command context: {err:?}");
}
engine_state
}
#[cfg(test)]
mod test_dataframe;

108
crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs
View file

@ -1,108 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct AllFalse;
impl Command for AllFalse {
fn name(&self) -> &str {
"dfr all-false"
}
fn usage(&self) -> &str {
"Returns true if all values are false."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Returns true if all values are false",
example: "[false false false] | dfr into-df | dfr all-false",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"all_false".to_string(),
vec![Value::test_bool(true)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Checks the result from a comparison",
example: r#"let s = ([5 6 2 10] | dfr into-df);
let res = ($s > 9);
$res | dfr all-false"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"all_false".to_string(),
vec![Value::test_bool(false)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let bool = series.bool().map_err(|_| ShellError::GenericError {
error: "Error converting to bool".into(),
msg: "all-false only works with series of type bool".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let value = Value::bool(!bool.any(), call.head);
NuDataFrame::try_from_columns(
vec![Column::new("all_false".to_string(), vec![value])],
None,
)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AllFalse {})])
}
}

105
crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs
View file

@ -1,105 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct AllTrue;
impl Command for AllTrue {
fn name(&self) -> &str {
"dfr all-true"
}
fn usage(&self) -> &str {
"Returns true if all values are true."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Returns true if all values are true",
example: "[true true true] | dfr into-df | dfr all-true",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"all_true".to_string(),
vec![Value::test_bool(true)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Checks the result from a comparison",
example: r#"let s = ([5 6 2 8] | dfr into-df);
let res = ($s > 9);
$res | dfr all-true"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"all_true".to_string(),
vec![Value::test_bool(false)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let bool = series.bool().map_err(|_| ShellError::GenericError {
error: "Error converting to bool".into(),
msg: "all-false only works with series of type bool".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let value = Value::bool(bool.all(), call.head);
NuDataFrame::try_from_columns(vec![Column::new("all_true".to_string(), vec![value])], None)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AllTrue {})])
}
}

85
crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs
View file

@ -1,85 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked};
#[derive(Clone)]
pub struct ArgMax;
impl Command for ArgMax {
fn name(&self) -> &str {
"dfr arg-max"
}
fn usage(&self) -> &str {
"Return index for max value in series."
}
fn search_terms(&self) -> Vec<&str> {
vec!["argmax", "maximum", "most", "largest", "greatest"]
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns index for max value",
example: "[1 3 2] | dfr into-df | dfr arg-max",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new("arg_max".to_string(), vec![Value::test_int(1)])],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let res = series.arg_max();
let chunked = match res {
Some(index) => UInt32Chunked::from_slice("arg_max", &[index as u32]),
None => UInt32Chunked::from_slice("arg_max", &[]),
};
let res = chunked.into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ArgMax {})])
}
}

85
crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs
View file

@ -1,85 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked};
#[derive(Clone)]
pub struct ArgMin;
impl Command for ArgMin {
fn name(&self) -> &str {
"dfr arg-min"
}
fn usage(&self) -> &str {
"Return index for min value in series."
}
fn search_terms(&self) -> Vec<&str> {
vec!["argmin", "minimum", "least", "smallest", "lowest"]
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns index for min value",
example: "[1 3 2] | dfr into-df | dfr arg-min",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new("arg_min".to_string(), vec![Value::test_int(0)])],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let res = series.arg_min();
let chunked = match res {
Some(index) => UInt32Chunked::from_slice("arg_min", &[index as u32]),
None => UInt32Chunked::from_slice("arg_min", &[]),
};
let res = chunked.into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ArgMin {})])
}
}

148
crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs
View file

@ -1,148 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DataType, IntoSeries};
use polars_ops::prelude::{cum_max, cum_min, cum_sum};
enum CumType {
Min,
Max,
Sum,
}
impl CumType {
fn from_str(roll_type: &str, span: Span) -> Result<Self, ShellError> {
match roll_type {
"min" => Ok(Self::Min),
"max" => Ok(Self::Max),
"sum" => Ok(Self::Sum),
_ => Err(ShellError::GenericError {
error: "Wrong operation".into(),
msg: "Operation not valid for cumulative".into(),
span: Some(span),
help: Some("Allowed values: max, min, sum".into()),
inner: vec![],
}),
}
}
fn to_str(&self) -> &'static str {
match self {
CumType::Min => "cumulative_min",
CumType::Max => "cumulative_max",
CumType::Sum => "cumulative_sum",
}
}
}
#[derive(Clone)]
pub struct Cumulative;
impl Command for Cumulative {
fn name(&self) -> &str {
"dfr cumulative"
}
fn usage(&self) -> &str {
"Cumulative calculation for a series."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("type", SyntaxShape::String, "rolling operation")
.switch("reverse", "Reverse cumulative calculation", Some('r'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Cumulative sum for a series",
example: "[1 2 3 4 5] | dfr into-df | dfr cumulative sum",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0_cumulative_sum".to_string(),
vec![
Value::test_int(1),
Value::test_int(3),
Value::test_int(6),
Value::test_int(10),
Value::test_int(15),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let cum_type: Spanned<String> = call.req(engine_state, stack, 0)?;
let reverse = call.has_flag(engine_state, stack, "reverse")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
if let DataType::Object(..) = series.dtype() {
return Err(ShellError::GenericError {
error: "Found object series".into(),
msg: "Series of type object cannot be used for cumulative operation".into(),
span: Some(call.head),
help: None,
inner: vec![],
});
}
let cum_type = CumType::from_str(&cum_type.item, cum_type.span)?;
let mut res = match cum_type {
CumType::Max => cum_max(&series, reverse),
CumType::Min => cum_min(&series, reverse),
CumType::Sum => cum_sum(&series, reverse),
}
.map_err(|e| ShellError::GenericError {
error: "Error creating cumulative".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let name = format!("{}_{}", series.name(), cum_type.to_str());
res.rename(&name);
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Cumulative {})])
}
}

94
crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs
View file

@ -1,94 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{IntoSeries, StringMethods};
#[derive(Clone)]
pub struct AsDate;
impl Command for AsDate {
fn name(&self) -> &str {
"dfr as-date"
}
fn usage(&self) -> &str {
r#"Converts string to date."#
}
fn extra_usage(&self) -> &str {
r#"Format example:
"%Y-%m-%d" => 2021-12-31
"%d-%m-%Y" => 31-12-2021
"%Y%m%d" => 2021319 (2021-03-19)"#
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("format", SyntaxShape::String, "formatting date string")
.switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Converts string to date",
example: r#"["2021-12-30" "2021-12-31"] | dfr into-df | dfr as-datetime "%Y-%m-%d""#,
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let format: String = call.req(engine_state, stack, 0)?;
let not_exact = call.has_flag(engine_state, stack, "not-exact")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.str().map_err(|e| ShellError::GenericError {
error: "Error casting to string".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = if not_exact {
casted.as_date_not_exact(Some(format.as_str()))
} else {
casted.as_date(Some(format.as_str()), false)
};
let mut res = res
.map_err(|e| ShellError::GenericError {
error: "Error creating datetime".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into_series();
res.rename("date");
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}

187
crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs
View file

@ -1,187 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use chrono::DateTime;
use nu_engine::command_prelude::*;
use polars::prelude::{IntoSeries, StringMethods, TimeUnit};
#[derive(Clone)]
pub struct AsDateTime;
impl Command for AsDateTime {
fn name(&self) -> &str {
"dfr as-datetime"
}
fn usage(&self) -> &str {
r#"Converts string to datetime."#
}
fn extra_usage(&self) -> &str {
r#"Format example:
"%y/%m/%d %H:%M:%S" => 21/12/31 12:54:98
"%y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01
"%y/%m/%d %H:%M:%S" => 21/12/31 24:58:01
"%y%m%d %H:%M:%S" => 210319 23:58:50
"%Y/%m/%d %H:%M:%S" => 2021/12/31 12:54:98
"%Y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01
"%Y/%m/%d %H:%M:%S" => 2021/12/31 24:58:01
"%Y%m%d %H:%M:%S" => 20210319 23:58:50
"%FT%H:%M:%S" => 2019-04-18T02:45:55
"%FT%H:%M:%S.%6f" => microseconds
"%FT%H:%M:%S.%9f" => nanoseconds"#
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("format", SyntaxShape::String, "formatting date time string")
.switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Converts string to datetime",
example: r#"["2021-12-30 00:00:00" "2021-12-31 00:00:00"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S""#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"datetime".to_string(),
vec![
Value::date(
DateTime::parse_from_str(
"2021-12-30 00:00:00 +0000",
"%Y-%m-%d %H:%M:%S %z",
)
.expect("date calculation should not fail in test"),
Span::test_data(),
),
Value::date(
DateTime::parse_from_str(
"2021-12-31 00:00:00 +0000",
"%Y-%m-%d %H:%M:%S %z",
)
.expect("date calculation should not fail in test"),
Span::test_data(),
),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Converts string to datetime with high resolutions",
example: r#"["2021-12-30 00:00:00.123456789" "2021-12-31 00:00:00.123456789"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S.%9f""#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"datetime".to_string(),
vec![
Value::date(
DateTime::parse_from_str(
"2021-12-30 00:00:00.123456789 +0000",
"%Y-%m-%d %H:%M:%S.%9f %z",
)
.expect("date calculation should not fail in test"),
Span::test_data(),
),
Value::date(
DateTime::parse_from_str(
"2021-12-31 00:00:00.123456789 +0000",
"%Y-%m-%d %H:%M:%S.%9f %z",
)
.expect("date calculation should not fail in test"),
Span::test_data(),
),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let format: String = call.req(engine_state, stack, 0)?;
let not_exact = call.has_flag(engine_state, stack, "not-exact")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.str().map_err(|e| ShellError::GenericError {
error: "Error casting to string".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = if not_exact {
casted.as_datetime_not_exact(
Some(format.as_str()),
TimeUnit::Nanoseconds,
false,
None,
&Default::default(),
)
} else {
casted.as_datetime(
Some(format.as_str()),
TimeUnit::Nanoseconds,
false,
false,
None,
&Default::default(),
)
};
let mut res = res
.map_err(|e| ShellError::GenericError {
error: "Error creating datetime".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into_series();
res.rename("datetime");
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AsDateTime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetDay;
impl Command for GetDay {
fn name(&self) -> &str {
"dfr get-day"
}
fn usage(&self) -> &str {
"Gets day from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns day from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-day"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(4), Value::test_int(4)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.day().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetDay {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetHour;
impl Command for GetHour {
fn name(&self) -> &str {
"dfr get-hour"
}
fn usage(&self) -> &str {
"Gets hour from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns hour from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-hour"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(16), Value::test_int(16)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.hour().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetHour {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetMinute;
impl Command for GetMinute {
fn name(&self) -> &str {
"dfr get-minute"
}
fn usage(&self) -> &str {
"Gets minute from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns minute from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-minute"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(39), Value::test_int(39)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.minute().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetMinute {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetMonth;
impl Command for GetMonth {
fn name(&self) -> &str {
"dfr get-month"
}
fn usage(&self) -> &str {
"Gets month from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns month from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-month"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(8), Value::test_int(8)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.month().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetMonth {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetNanosecond;
impl Command for GetNanosecond {
fn name(&self) -> &str {
"dfr get-nanosecond"
}
fn usage(&self) -> &str {
"Gets nanosecond from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns nanosecond from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-nanosecond"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(0), Value::test_int(0)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.nanosecond().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetNanosecond {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetOrdinal;
impl Command for GetOrdinal {
fn name(&self) -> &str {
"dfr get-ordinal"
}
fn usage(&self) -> &str {
"Gets ordinal from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns ordinal from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-ordinal"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(217), Value::test_int(217)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.ordinal().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetOrdinal {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetSecond;
impl Command for GetSecond {
fn name(&self) -> &str {
"dfr get-second"
}
fn usage(&self) -> &str {
"Gets second from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns second from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-second"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(18), Value::test_int(18)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.second().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetSecond {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetWeek;
impl Command for GetWeek {
fn name(&self) -> &str {
"dfr get-week"
}
fn usage(&self) -> &str {
"Gets week from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns week from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-week"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(32), Value::test_int(32)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.week().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetWeek {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetWeekDay;
impl Command for GetWeekDay {
fn name(&self) -> &str {
"dfr get-weekday"
}
fn usage(&self) -> &str {
"Gets weekday from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns weekday from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-weekday"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.weekday().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetWeekDay {}), Box::new(IntoDatetime {})])
}
}

90
crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs
View file

@ -1,90 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{DatetimeMethods, IntoSeries};
#[derive(Clone)]
pub struct GetYear;
impl Command for GetYear {
fn name(&self) -> &str {
"dfr get-year"
}
fn usage(&self) -> &str {
"Gets year from date."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns year from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC');
let df = ([$dt $dt] | dfr into-df);
$df | dfr get-year"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(2020), Value::test_int(2020)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let casted = series.datetime().map_err(|e| ShellError::GenericError {
error: "Error casting to datetime type".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let res = casted.year().into_series();
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(explore_refactor_IntoDatetime)]
mod test {
use super::super::super::super::super::IntoDatetime;
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetYear {}), Box::new(IntoDatetime {})])
}
}

25
crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs
View file

@ -1,25 +0,0 @@
mod as_date;
mod as_datetime;
mod get_day;
mod get_hour;
mod get_minute;
mod get_month;
mod get_nanosecond;
mod get_ordinal;
mod get_second;
mod get_week;
mod get_weekday;
mod get_year;
pub use as_date::AsDate;
pub use as_datetime::AsDateTime;
pub use get_day::GetDay;
pub use get_hour::GetHour;
pub use get_minute::GetMinute;
pub use get_month::GetMonth;
pub use get_nanosecond::GetNanosecond;
pub use get_ordinal::GetOrdinal;
pub use get_second::GetSecond;
pub use get_week::GetWeek;
pub use get_weekday::GetWeekDay;
pub use get_year::GetYear;

130
crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs
View file

@ -1,130 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{IntoSeries, SortOptions};
#[derive(Clone)]
pub struct ArgSort;
impl Command for ArgSort {
fn name(&self) -> &str {
"dfr arg-sort"
}
fn usage(&self) -> &str {
"Returns indexes for a sorted series."
}
fn search_terms(&self) -> Vec<&str> {
vec!["argsort", "order", "arrange"]
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("reverse", "reverse order", Some('r'))
.switch("nulls-last", "nulls ordered last", Some('n'))
.switch(
"maintain-order",
"maintain order on sorted items",
Some('m'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Returns indexes for a sorted series",
example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"arg_sort".to_string(),
vec![
Value::test_int(0),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
Value::test_int(4),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Returns indexes for a sorted series",
example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort --reverse",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"arg_sort".to_string(),
vec![
Value::test_int(3),
Value::test_int(4),
Value::test_int(1),
Value::test_int(2),
Value::test_int(0),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let sort_options = SortOptions {
descending: call.has_flag(engine_state, stack, "reverse")?,
nulls_last: call.has_flag(engine_state, stack, "nulls-last")?,
multithreaded: true,
maintain_order: call.has_flag(engine_state, stack, "maintain-order")?,
};
let mut res = df
.as_series(call.head)?
.arg_sort(sort_options)
.into_series();
res.rename("arg_sort");
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ArgSort {})])
}
}

115
crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs
View file

@ -1,115 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{arg_where, col, IntoLazy};
#[derive(Clone)]
pub struct ArgTrue;
impl Command for ArgTrue {
fn name(&self) -> &str {
"dfr arg-true"
}
fn usage(&self) -> &str {
"Returns indexes where values are true."
}
fn search_terms(&self) -> Vec<&str> {
vec!["argtrue", "truth", "boolean-true"]
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns indexes where values are true",
example: "[false true false] | dfr into-df | dfr arg-true",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"arg_true".to_string(),
vec![Value::test_int(1)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let columns = df.as_ref().get_column_names();
if columns.len() > 1 {
return Err(ShellError::GenericError {
error: "Error using as series".into(),
msg: "dataframe has more than one column".into(),
span: Some(call.head),
help: None,
inner: vec![],
});
}
match columns.first() {
Some(column) => {
let expression = arg_where(col(column).eq(true)).alias("arg_true");
let res = df
.as_ref()
.clone()
.lazy()
.select(&[expression])
.collect()
.map_err(|err| ShellError::GenericError {
error: "Error creating index column".into(),
msg: err.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let value = NuDataFrame::dataframe_into_value(res, call.head);
Ok(PipelineData::Value(value, None))
}
_ => Err(ShellError::UnsupportedInput {
msg: "Expected the dataframe to have a column".to_string(),
input: "".to_string(),
msg_span: call.head,
input_span: call.head,
}),
}
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ArgTrue {})])
}
}

93
crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs
View file

@ -1,93 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::IntoSeries;
#[derive(Clone)]
pub struct ArgUnique;
impl Command for ArgUnique {
fn name(&self) -> &str {
"dfr arg-unique"
}
fn usage(&self) -> &str {
"Returns indexes for unique values."
}
fn search_terms(&self) -> Vec<&str> {
vec!["argunique", "distinct", "noduplicate", "unrepeated"]
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns indexes for unique values",
example: "[1 2 2 3 3] | dfr into-df | dfr arg-unique",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"arg_unique".to_string(),
vec![Value::test_int(0), Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut res = df
.as_series(call.head)?
.arg_unique()
.map_err(|e| ShellError::GenericError {
error: "Error extracting unique values".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into_series();
res.rename("arg_unique");
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ArgUnique {})])
}
}

9
crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs
View file

@ -1,9 +0,0 @@
mod arg_sort;
mod arg_true;
mod arg_unique;
mod set_with_idx;
pub use arg_sort::ArgSort;
pub use arg_true::ArgTrue;
pub use arg_unique::ArgUnique;
pub use set_with_idx::SetWithIndex;

213
crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs
View file

@ -1,213 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{ChunkSet, DataType, IntoSeries};
#[derive(Clone)]
pub struct SetWithIndex;
impl Command for SetWithIndex {
fn name(&self) -> &str {
"dfr set-with-idx"
}
fn usage(&self) -> &str {
"Sets value in the given index."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("value", SyntaxShape::Any, "value to be inserted in series")
.required_named(
"indices",
SyntaxShape::Any,
"list of indices indicating where to set the value",
Some('i'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Set value in selected rows from series",
example: r#"let series = ([4 1 5 2 4 3] | dfr into-df);
let indices = ([0 2] | dfr into-df);
$series | dfr set-with-idx 6 --indices $indices"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_int(6),
Value::test_int(1),
Value::test_int(6),
Value::test_int(2),
Value::test_int(4),
Value::test_int(3),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value: Value = call.req(engine_state, stack, 0)?;
let indices_value: Value = call
.get_flag(engine_state, stack, "indices")?
.expect("required named value");
let indices_span = indices_value.span();
let indices = NuDataFrame::try_from_value(indices_value)?.as_series(indices_span)?;
let casted = match indices.dtype() {
DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => indices
.as_ref()
.cast(&DataType::UInt32)
.map_err(|e| ShellError::GenericError {
error: "Error casting indices".into(),
msg: e.to_string(),
span: Some(indices_span),
help: None,
inner: vec![],
}),
_ => Err(ShellError::GenericError {
error: "Incorrect type".into(),
msg: "Series with incorrect type".into(),
span: Some(indices_span),
help: Some("Consider using a Series with type int type".into()),
inner: vec![],
}),
}?;
let indices = casted
.u32()
.map_err(|e| ShellError::GenericError {
error: "Error casting indices".into(),
msg: e.to_string(),
span: Some(indices_span),
help: None,
inner: vec![],
})?
.into_iter()
.flatten();
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let series = df.as_series(call.head)?;
let span = value.span();
let res = match value {
Value::Int { val, .. } => {
let chunked = series.i64().map_err(|e| ShellError::GenericError {
error: "Error casting to i64".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
})?;
let res = chunked.scatter_single(indices, Some(val)).map_err(|e| {
ShellError::GenericError {
error: "Error setting value".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
}
})?;
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
}
Value::Float { val, .. } => {
let chunked = series.f64().map_err(|e| ShellError::GenericError {
error: "Error casting to f64".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
})?;
let res = chunked.scatter_single(indices, Some(val)).map_err(|e| {
ShellError::GenericError {
error: "Error setting value".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
}
})?;
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
}
Value::String { val, .. } => {
let chunked = series.str().map_err(|e| ShellError::GenericError {
error: "Error casting to string".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
})?;
let res = chunked
.scatter_single(indices, Some(val.as_ref()))
.map_err(|e| ShellError::GenericError {
error: "Error setting value".into(),
msg: e.to_string(),
span: Some(span),
help: None,
inner: vec![],
})?;
let mut res = res.into_series();
res.rename("string");
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
}
_ => Err(ShellError::GenericError {
error: "Incorrect value type".into(),
msg: format!(
"this value cannot be set in a series of type '{}'",
series.dtype()
),
span: Some(span),
help: None,
inner: vec![],
}),
};
res.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SetWithIndex {})])
}
}

122
crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs
View file

@ -1,122 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::IntoSeries;
#[derive(Clone)]
pub struct IsDuplicated;
impl Command for IsDuplicated {
fn name(&self) -> &str {
"dfr is-duplicated"
}
fn usage(&self) -> &str {
"Creates mask indicating duplicated values."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create mask indicating duplicated values",
example: "[5 6 6 6 8 8 8] | dfr into-df | dfr is-duplicated",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"is_duplicated".to_string(),
vec![
Value::test_bool(false),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Create mask indicating duplicated rows in a dataframe",
example:
"[[a, b]; [1 2] [1 2] [3 3] [3 3] [1 1]] | dfr into-df | dfr is-duplicated",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"is_duplicated".to_string(),
vec![
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut res = df
.as_ref()
.is_duplicated()
.map_err(|e| ShellError::GenericError {
error: "Error finding duplicates".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into_series();
res.rename("is_duplicated");
NuDataFrame::try_from_series(vec![res], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(IsDuplicated {})])
}
}

104
crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs
View file

@ -1,104 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::{is_in, IntoSeries};
#[derive(Clone)]
pub struct IsIn;
impl Command for IsIn {
fn name(&self) -> &str {
"dfr is-in"
}
fn usage(&self) -> &str {
"Checks if elements from a series are contained in right series."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("other", SyntaxShape::Any, "right series")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Checks if elements from a series are contained in right series",
example: r#"let other = ([1 3 6] | dfr into-df);
[5 6 6 6 8 8 8] | dfr into-df | dfr is-in $other"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"is_in".to_string(),
vec![
Value::test_bool(false),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
Value::test_bool(false),
Value::test_bool(false),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?.as_series(call.head)?;
let other_value: Value = call.req(engine_state, stack, 0)?;
let other_span = other_value.span();
let other_df = NuDataFrame::try_from_value(other_value)?;
let other = other_df.as_series(other_span)?;
let mut res = is_in(&df, &other)
.map_err(|e| ShellError::GenericError {
error: "Error finding in other".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into_series();
res.rename("is_in");
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(IsIn {})])
}
}

122
crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs
View file

@ -1,122 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::IntoSeries;
#[derive(Clone)]
pub struct IsNotNull;
impl Command for IsNotNull {
fn name(&self) -> &str {
"dfr is-not-null"
}
fn usage(&self) -> &str {
"Creates mask where value is not null."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create mask where values are not null",
example: r#"let s = ([5 6 0 8] | dfr into-df);
let res = ($s / $s);
$res | dfr is-not-null"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"is_not_null".to_string(),
vec![
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
Value::test_bool(true),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a is not null expression from a column",
example: "dfr col a | dfr is-not-null",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().is_not_null().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mut res = df.as_series(call.head)?.is_not_null();
res.rename("is_not_null");
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example};
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(IsNotNull {})]);
test_dataframe_example(&mut engine_state, &IsNotNull.examples()[0]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(IsNotNull {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &IsNotNull.examples()[1]);
}
}

122
crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs
View file

@ -1,122 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::IntoSeries;
#[derive(Clone)]
pub struct IsNull;
impl Command for IsNull {
fn name(&self) -> &str {
"dfr is-null"
}
fn usage(&self) -> &str {
"Creates mask where value is null."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create mask where values are null",
example: r#"let s = ([5 6 0 8] | dfr into-df);
let res = ($s / $s);
$res | dfr is-null"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"is_null".to_string(),
vec![
Value::test_bool(false),
Value::test_bool(false),
Value::test_bool(true),
Value::test_bool(false),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a is null expression from a column",
example: "dfr col a | dfr is-null",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().is_null().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mut res = df.as_series(call.head)?.is_null();
res.rename("is_null");
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example};
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(IsNull {})]);
test_dataframe_example(&mut engine_state, &IsNull.examples()[0]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(IsNull {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &IsNull.examples()[1]);
}
}

Some files were not shown because too many files have changed in this diff Show more