Move filesize to use bigint (#2984)

* Move filesize to be bigint-sized

* Add tests and fix filesize display

* clippy

crates/nu-command/src/commands/autoview/command.rs

@@ -173,6 +173,13 @@ pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellErr
                         let output = format_leaf(&x).plain_string(100_000);
                         out!("{}", output);
                     }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Filesize(_)),
+                        ..
+                    } => {
+                        let output = format_leaf(&x).plain_string(100_000);
+                        out!("{}", output);
+                    }
                     Value {
                         value: UntaggedValue::Primitive(Primitive::Date(d)),
                         ..

crates/nu-command/src/commands/format/format_filesize.rs

@@ -108,9 +108,12 @@ fn convert_bytes_to_string_using_format(
 ) -> Result<Value, ShellError> {
     match bytes.value {
         Primitive(Filesize(b)) => {
-            let byte = byte_unit::Byte::from_bytes(b as u128);
+            if let Some(value) = b.to_u128() {
+                let byte = byte_unit::Byte::from_bytes(value);
                 let value = match format.item().to_lowercase().as_str() {
-                "b" => Ok(UntaggedValue::string(b.to_formatted_string(&Locale::en))),
+                    "b" => Ok(UntaggedValue::string(
+                        value.to_formatted_string(&Locale::en),
+                    )),
                     "kb" => Ok(UntaggedValue::string(
                         byte.get_adjusted_unit(byte_unit::ByteUnit::KB).to_string(),
                     )),
@@ -163,6 +166,13 @@ fn convert_bytes_to_string_using_format(
                     Ok(b) => Ok(Value { value: b, ..bytes }),
                     Err(e) => Err(e),
                 }
+            } else {
+                Err(ShellError::labeled_error(
+                    "Value too large to fit in 128 bits",
+                    "value too large to fit in format",
+                    format.span(),
+                ))
+            }
         }
         _ => Err(ShellError::labeled_error(
             "the data in this row is not of the type filesize",

crates/nu-command/src/commands/math/avg.rs

@@ -59,12 +59,9 @@ impl WholeStreamCommand for SubCommand {
 
 fn to_byte(value: &Value) -> Option<Value> {
     match &value.value {
-        UntaggedValue::Primitive(Primitive::Int(num)) => Some(
-            UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
-                num.clone(),
-            ))))
-            .into_untagged_value(),
-        ),
+        UntaggedValue::Primitive(Primitive::Int(num)) => {
+            Some(UntaggedValue::Primitive(Primitive::Filesize(num.clone())).into_untagged_value())
+        }
         _ => None,
     }
 }
@@ -95,7 +92,7 @@ pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
                     Value {
                         value: UntaggedValue::Primitive(Primitive::Filesize(num)),
                         ..
-                    } => UntaggedValue::int(*num as usize).into_untagged_value(),
+                    } => UntaggedValue::int(num.clone()).into_untagged_value(),
                     other => other.clone(),
                 })
                 .collect::<Vec<_>>(),
@@ -116,7 +113,7 @@ pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
             value: UntaggedValue::Primitive(Primitive::Filesize(num)),
             ..
         } => {
-            let left = UntaggedValue::from(Primitive::Int(num.into()));
+            let left = UntaggedValue::from(Primitive::Int(num));
             let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {

crates/nu-command/src/commands/math/median.rs

@@ -137,7 +137,7 @@ fn compute_average(values: &[Value], name: impl Into<Tag>) -> Result<Value, Shel
             value: UntaggedValue::Primitive(Primitive::Filesize(num)),
             ..
         } => {
-            let left = UntaggedValue::from(Primitive::Int(num.into()));
+            let left = UntaggedValue::from(Primitive::Int(num));
             let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {

crates/nu-command/src/commands/math/product.rs

@@ -3,10 +3,7 @@ use crate::commands::math::utils::run_with_function;
 use crate::prelude::*;
 use nu_engine::WholeStreamCommand;
 use nu_errors::ShellError;
-use nu_protocol::{
-    hir::{convert_number_to_u64, Number},
-    Primitive, Signature, UntaggedValue, Value,
-};
+use nu_protocol::{Primitive, Signature, UntaggedValue, Value};
 
 pub struct SubCommand;
 
@@ -51,12 +48,9 @@ impl WholeStreamCommand for SubCommand {
 
 fn to_byte(value: &Value) -> Option<Value> {
     match &value.value {
-        UntaggedValue::Primitive(Primitive::Int(num)) => Some(
-            UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
-                num.clone(),
-            ))))
-            .into_untagged_value(),
-        ),
+        UntaggedValue::Primitive(Primitive::Int(num)) => {
+            Some(UntaggedValue::Primitive(Primitive::Filesize(num.clone())).into_untagged_value())
+        }
         _ => None,
     }
 }
@@ -78,7 +72,7 @@ pub fn product(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
                     Value {
                         value: UntaggedValue::Primitive(Primitive::Filesize(num)),
                         ..
-                    } => UntaggedValue::int(*num as usize).into_untagged_value(),
+                    } => UntaggedValue::int(num.clone()).into_untagged_value(),
                     other => other.clone(),
                 })
                 .collect::<Vec<_>>(),

crates/nu-command/src/commands/math/sum.rs

@@ -4,10 +4,7 @@ use crate::prelude::*;
 use nu_engine::WholeStreamCommand;
 use nu_errors::ShellError;
 
-use nu_protocol::{
-    hir::{convert_number_to_u64, Number},
-    Primitive, Signature, UntaggedValue, Value,
-};
+use nu_protocol::{Primitive, Signature, UntaggedValue, Value};
 
 pub struct SubCommand;
 
@@ -59,12 +56,9 @@ impl WholeStreamCommand for SubCommand {
 
 fn to_byte(value: &Value) -> Option<Value> {
     match &value.value {
-        UntaggedValue::Primitive(Primitive::Int(num)) => Some(
-            UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
-                num.clone(),
-            ))))
-            .into_untagged_value(),
-        ),
+        UntaggedValue::Primitive(Primitive::Int(num)) => {
+            Some(UntaggedValue::Primitive(Primitive::Filesize(num.clone())).into_untagged_value())
+        }
         _ => None,
     }
 }
@@ -90,7 +84,7 @@ pub fn summation(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
                     Value {
                         value: UntaggedValue::Primitive(Primitive::Filesize(num)),
                         ..
-                    } => UntaggedValue::int(*num as usize).into_untagged_value(),
+                    } => UntaggedValue::int(num.clone()).into_untagged_value(),
                     other => other.clone(),
                 })
                 .collect::<Vec<_>>(),

crates/nu-command/src/commands/math/variance.rs

@@ -130,7 +130,7 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
                 value: UntaggedValue::Primitive(Primitive::Filesize(num)),
                 ..
             } => {
-                UntaggedValue::from(Primitive::Int(num.clone().into()))
+                UntaggedValue::from(Primitive::Int(num.clone()))
             },
             Value {
                 value: UntaggedValue::Primitive(num),

crates/nu-command/src/commands/to_delimited_data.rs

@@ -116,7 +116,7 @@ pub fn clone_tagged_value(v: &Value) -> Value {
             UntaggedValue::Primitive(Primitive::FilePath(x.clone()))
         }
         UntaggedValue::Primitive(Primitive::Filesize(b)) => {
-            UntaggedValue::Primitive(Primitive::Filesize(*b))
+            UntaggedValue::Primitive(Primitive::Filesize(b.clone()))
         }
         UntaggedValue::Primitive(Primitive::Date(d)) => {
             UntaggedValue::Primitive(Primitive::Date(*d))

crates/nu-command/src/commands/to_toml.rs

@@ -40,7 +40,17 @@ impl WholeStreamCommand for ToTOML {
 fn helper(v: &Value) -> Result<toml::Value, ShellError> {
     Ok(match &v.value {
         UntaggedValue::Primitive(Primitive::Boolean(b)) => toml::Value::Boolean(*b),
-        UntaggedValue::Primitive(Primitive::Filesize(b)) => toml::Value::Integer(*b as i64),
+        UntaggedValue::Primitive(Primitive::Filesize(b)) => {
+            if let Some(value) = b.to_i64() {
+                toml::Value::Integer(value)
+            } else {
+                return Err(ShellError::labeled_error(
+                    "Value too large to write to toml",
+                    "value too large for toml",
+                    v.tag.span,
+                ));
+            }
+        }
         UntaggedValue::Primitive(Primitive::Duration(i)) => toml::Value::String(i.to_string()),
         UntaggedValue::Primitive(Primitive::Date(d)) => toml::Value::String(d.to_string()),
         UntaggedValue::Primitive(Primitive::EndOfStream) => {

crates/nu-data/src/base.rs

@@ -129,20 +129,18 @@ pub fn coerce_compare_primitive(
         (Int(left), Decimal(right)) => {
             CompareValues::Decimals(BigDecimal::zero() + left, right.clone())
         }
-        (Int(left), Filesize(right)) => CompareValues::Ints(left.clone(), BigInt::from(*right)),
+        (Int(left), Filesize(right)) => CompareValues::Ints(left.clone(), right.clone()),
         (Decimal(left), Decimal(right)) => CompareValues::Decimals(left.clone(), right.clone()),
         (Decimal(left), Int(right)) => {
             CompareValues::Decimals(left.clone(), BigDecimal::zero() + right)
         }
         (Decimal(left), Filesize(right)) => {
-            CompareValues::Decimals(left.clone(), BigDecimal::from(*right))
+            CompareValues::Decimals(left.clone(), BigDecimal::from(right.clone()))
         }
-        (Filesize(left), Filesize(right)) => {
-            CompareValues::Ints(BigInt::from(*left), BigInt::from(*right))
-        }
-        (Filesize(left), Int(right)) => CompareValues::Ints(BigInt::from(*left), right.clone()),
+        (Filesize(left), Filesize(right)) => CompareValues::Ints(left.clone(), right.clone()),
+        (Filesize(left), Int(right)) => CompareValues::Ints(left.clone(), right.clone()),
         (Filesize(left), Decimal(right)) => {
-            CompareValues::Decimals(BigDecimal::from(*left), right.clone())
+            CompareValues::Decimals(BigDecimal::from(left.clone()), right.clone())
         }
         (Nothing, Nothing) => CompareValues::Booleans(true, true),
         (String(left), String(right)) => CompareValues::String(left.clone(), right.clone()),

crates/nu-data/src/base/shape.rs

@@ -24,7 +24,7 @@ pub enum InlineShape {
     Int(BigInt),
     Decimal(BigDecimal),
     Range(Box<InlineRange>),
-    Bytesize(u64),
+    Bytesize(BigInt),
     String(String),
     Line(String),
     ColumnPath(ColumnPath),
@@ -68,7 +68,7 @@ impl InlineShape {
                 }))
             }
             Primitive::Decimal(decimal) => InlineShape::Decimal(decimal.clone()),
-            Primitive::Filesize(bytesize) => InlineShape::Bytesize(*bytesize),
+            Primitive::Filesize(bytesize) => InlineShape::Bytesize(bytesize.clone()),
             Primitive::String(string) => InlineShape::String(string.clone()),
             Primitive::ColumnPath(path) => InlineShape::ColumnPath(path.clone()),
             Primitive::GlobPattern(pattern) => InlineShape::GlobPattern(pattern.clone()),
@@ -128,7 +128,9 @@ impl InlineShape {
         }
     }
 
-    pub fn format_bytes(bytesize: &u64) -> (DbgDocBldr, String) {
+    pub fn format_bytes(bytesize: &BigInt) -> (DbgDocBldr, String) {
+        use bigdecimal::ToPrimitive;
+
         // get the config value, if it doesn't exist make it 'auto' so it works how it originally did
         let filesize_format_var = crate::config::config(Tag::unknown())
             .expect("unable to get the config.toml file")
@@ -155,8 +157,10 @@ impl InlineShape {
             _ => (byte_unit::ByteUnit::B, "auto"),
         };
 
-        let byte = byte_unit::Byte::from_bytes(*bytesize as u128);
-        let byte = if filesize_format.0 == byte_unit::ByteUnit::B && filesize_format.1 == "auto" {
+        if let Some(value) = bytesize.to_u128() {
+            let byte = byte_unit::Byte::from_bytes(value);
+            let byte = if filesize_format.0 == byte_unit::ByteUnit::B && filesize_format.1 == "auto"
+            {
                 byte.get_appropriate_unit(false)
             } else {
                 byte.get_adjusted_unit(filesize_format.0)
@@ -182,6 +186,13 @@ impl InlineShape {
                     (doc.clone(), InlineShape::render_doc(&doc))
                 }
             }
+        } else {
+            let doc = (DbgDocBldr::primitive(format!("{}", bytesize))
+                + DbgDocBldr::space()
+                + DbgDocBldr::kind("B"))
+            .group();
+            (doc.clone(), InlineShape::render_doc(&doc))
+        }
     }
 
     pub fn render_doc(doc: &DebugDocBuilder) -> String {

crates/nu-data/src/config.rs

@@ -61,9 +61,21 @@ fn collect_values(input: &[Value]) -> Result<Vec<toml::Value>, ShellError> {
 // Helper method to recursively convert nu_protocol::Value -> toml::Value
 // This shouldn't be called at the top-level
 fn helper(v: &Value) -> Result<toml::Value, ShellError> {
+    use bigdecimal::ToPrimitive;
+
     Ok(match &v.value {
         UntaggedValue::Primitive(Primitive::Boolean(b)) => toml::Value::Boolean(*b),
-        UntaggedValue::Primitive(Primitive::Filesize(b)) => toml::Value::Integer(*b as i64),
+        UntaggedValue::Primitive(Primitive::Filesize(b)) => {
+            if let Some(value) = b.to_i64() {
+                toml::Value::Integer(value)
+            } else {
+                return Err(ShellError::labeled_error(
+                    "Value too large to convert to toml value",
+                    "value too large",
+                    v.tag.span,
+                ));
+            }
+        }
         UntaggedValue::Primitive(Primitive::Duration(i)) => toml::Value::String(i.to_string()),
         UntaggedValue::Primitive(Primitive::Date(d)) => toml::Value::String(d.to_string()),
         UntaggedValue::Primitive(Primitive::EndOfStream) => {

crates/nu-data/src/value.rs

@@ -81,21 +81,12 @@ pub fn unsafe_compute_values(
     match (left, right) {
         (UntaggedValue::Primitive(lhs), UntaggedValue::Primitive(rhs)) => match (lhs, rhs) {
             (Primitive::Filesize(x), Primitive::Int(y)) => match operator {
-                Operator::Plus => Ok(UntaggedValue::Primitive(Primitive::Int(x + y))),
-                Operator::Minus => Ok(UntaggedValue::Primitive(Primitive::Int(x - y))),
-                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Int(x * y))),
-                Operator::Divide => Ok(UntaggedValue::Primitive(Primitive::Decimal(
-                    bigdecimal::BigDecimal::from(*x) / bigdecimal::BigDecimal::from(y.clone()),
-                ))),
+                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Filesize(x * y))),
+                Operator::Divide => Ok(UntaggedValue::Primitive(Primitive::Filesize(x / y))),
                 _ => Err((left.type_name(), right.type_name())),
             },
             (Primitive::Int(x), Primitive::Filesize(y)) => match operator {
-                Operator::Plus => Ok(UntaggedValue::Primitive(Primitive::Int(x + y))),
-                Operator::Minus => Ok(UntaggedValue::Primitive(Primitive::Int(x - y))),
-                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Int(x * y))),
-                Operator::Divide => Ok(UntaggedValue::Primitive(Primitive::Decimal(
-                    bigdecimal::BigDecimal::from(x.clone()) / bigdecimal::BigDecimal::from(*y),
-                ))),
+                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Filesize(x * y))),
                 _ => Err((left.type_name(), right.type_name())),
             },
             _ => Err((left.type_name(), right.type_name())),
@@ -120,8 +111,12 @@ pub fn compute_values(
                 Ok(UntaggedValue::Primitive(Primitive::Filesize(result)))
             }
             (Primitive::Filesize(x), Primitive::Int(y)) => match operator {
-                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Int(x * y))),
-                Operator::Divide => Ok(UntaggedValue::Primitive(Primitive::Int(x / y))),
+                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Filesize(x * y))),
+                Operator::Divide => Ok(UntaggedValue::Primitive(Primitive::Filesize(x / y))),
+                _ => Err((left.type_name(), right.type_name())),
+            },
+            (Primitive::Int(x), Primitive::Filesize(y)) => match operator {
+                Operator::Multiply => Ok(UntaggedValue::Primitive(Primitive::Filesize(x * y))),
                 _ => Err((left.type_name(), right.type_name())),
             },
             (Primitive::Int(x), Primitive::Int(y)) => match operator {

crates/nu-protocol/src/hir.rs

@@ -631,8 +631,8 @@ impl Unit {
     }
 }
 
-pub fn filesize(size_in_bytes: u64) -> UntaggedValue {
-    UntaggedValue::Primitive(Primitive::Filesize(size_in_bytes))
+pub fn filesize(size_in_bytes: impl Into<BigInt>) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Filesize(size_in_bytes.into()))
 }
 
 pub fn duration(nanos: BigInt) -> UntaggedValue {

crates/nu-protocol/src/value.rs

@@ -186,7 +186,7 @@ impl UntaggedValue {
     }
 
     /// Helper for creating filesize values
-    pub fn filesize(s: impl Into<u64>) -> UntaggedValue {
+    pub fn filesize(s: impl Into<BigInt>) -> UntaggedValue {
         UntaggedValue::Primitive(Primitive::Filesize(s.into()))
     }
 
@@ -713,7 +713,7 @@ impl U64Ext for u64 {
 
     fn to_filesize_value(&self, the_tag: Tag) -> Value {
         Value {
-            value: UntaggedValue::Primitive(Primitive::Filesize(*self)),
+            value: UntaggedValue::Primitive(Primitive::Filesize(BigInt::from(*self))),
             tag: the_tag,
         }
     }

crates/nu-protocol/src/value/primitive.rs

@@ -31,7 +31,7 @@ pub enum Primitive {
     #[serde(with = "serde_bigdecimal")]
     Decimal(BigDecimal),
     /// A count in the number of bytes, used as a filesize
-    Filesize(u64),
+    Filesize(BigInt),
     /// A string value
     String(String),
     /// A path to travel to reach a value in a table
@@ -254,7 +254,8 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S
         Primitive::EndOfStream => String::new(),
         Primitive::FilePath(p) => format!("{}", p.display()),
         Primitive::Filesize(num_bytes) => {
-            let byte = byte_unit::Byte::from_bytes(*num_bytes as u128);
+            if let Some(value) = num_bytes.to_u128() {
+                let byte = byte_unit::Byte::from_bytes(value);
 
                 if byte.get_bytes() == 0u128 {
                     return "—".to_string();
@@ -266,6 +267,9 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S
                     byte_unit::ByteUnit::B => format!("{} B ", byte.get_value()),
                     _ => byte.format(1),
                 }
+            } else {
+                format!("{} B", num_bytes)
+            }
         }
         Primitive::Duration(duration) => format_duration(duration),
         Primitive::Int(i) => i.to_string(),

tests/shell/pipeline/commands/internal.rs

@@ -750,6 +750,53 @@ fn range_with_mixed_types() {
     assert_eq!(actual.out, "55");
 }
 
+#[test]
+fn filesize_math() {
+    let actual = nu!(
+        cwd: ".",
+        r#"
+        = 100 * 10kb
+        "#
+    );
+
+    assert_eq!(actual.out, "1.0 MB");
+}
+
+#[test]
+fn filesize_math2() {
+    let actual = nu!(
+        cwd: ".",
+        r#"
+        = 100 / 10kb
+        "#
+    );
+
+    assert!(actual.err.contains("Coercion"));
+}
+
+#[test]
+fn filesize_math3() {
+    let actual = nu!(
+        cwd: ".",
+        r#"
+        = 100kb / 10
+        "#
+    );
+
+    assert_eq!(actual.out, "10.2 KB");
+}
+#[test]
+fn filesize_math4() {
+    let actual = nu!(
+        cwd: ".",
+        r#"
+        = 100kb * 5
+        "#
+    );
+
+    assert_eq!(actual.out, "512.0 KB");
+}
+
 #[test]
 fn exclusive_range_with_mixed_types() {
     let actual = nu!(