Implement tokenizer for GNU definitions.units

src/ast.rs

@@ -0,0 +1,75 @@
+use std::rc::Rc;
+
+#[derive(Debug, Clone)]
+pub enum SuffixOp {
+    Celsius,
+    Fahrenheit,
+    Reaumur,
+    Romer,
+    Delisle,
+    Newton,
+}
+
+#[derive(Debug, Clone)]
+pub enum DateToken {
+    Literal(String),
+    Number(String, Option<String>, Option<String>),
+    Colon,
+    Dash,
+    Space,
+    Plus,
+    Error(String),
+}
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Unit(String),
+    Quote(String),
+    Const(String, Option<String>, Option<String>),
+    Date(Vec<DateToken>),
+    Frac(Box<Expr>, Box<Expr>),
+    Mul(Vec<Expr>),
+    Pow(Box<Expr>, Box<Expr>),
+    Add(Box<Expr>, Box<Expr>),
+    Sub(Box<Expr>, Box<Expr>),
+    Neg(Box<Expr>),
+    Plus(Box<Expr>),
+    Convert(Box<Expr>, Box<Expr>),
+    Equals(Box<Expr>, Box<Expr>),
+    Suffix(SuffixOp, Box<Expr>),
+    Call(String, Vec<Expr>),
+    Factorize(Box<Expr>),
+    DegC,
+    DegF,
+    DegRe,
+    DegRo,
+    DegDe,
+    DegN,
+    Error(String),
+}
+
+#[derive(Debug, Clone)]
+pub enum DatePattern {
+    Literal(String),
+    Match(String),
+    Optional(Vec<DatePattern>),
+    Dash,
+    Colon,
+    Error(String),
+}
+
+#[derive(Debug)]
+pub enum Def {
+    Dimension(String),
+    Prefix(Expr),
+    SPrefix(Expr),
+    Unit(Expr),
+    DatePattern(Vec<DatePattern>),
+    Error(String),
+}
+
+#[derive(Debug)]
+pub struct Defs {
+    pub defs: Vec<(String, Rc<Def>)>,
+    pub aliases: Vec<(Expr, String)>,
+}

src/bin/rink-debug-tokens.rs

@@ -1,17 +1,43 @@
 extern crate rink;
 
-use rink::*;
+use rink::gnu_units::*;
 
 fn main() {
     use std::io::Read;
     use std::fs::File;
 
-    let mut f = File::open("units.txt").unwrap();
+    let mut f = File::open("definitions.units").unwrap();
     let mut buf = vec![];
     f.read_to_end(&mut buf).unwrap();
     let string = String::from_utf8_lossy(&*buf);
-    let mut iter = unit_defs::TokenIterator::new(&*string).peekable();
-    let res = unit_defs::tokens(&mut iter);
+    let mut iter = TokenIterator::new(&*string).peekable();
+    let res = tokens(&mut iter);
 
-    println!("{:#?}", res);
+    for tok in res {
+        match tok {
+            Token::Eof => panic!(),
+            Token::Newline => print!("\n"),
+            Token::Ident(name) => print!("`{}` ", name),
+            Token::Number(i, f, e) => {
+                print!("{}", i);
+                if let Some(f) = f {
+                    print!(".{}", f);
+                }
+                if let Some(e) = e {
+                    print!("e{}", e);
+                }
+                print!(" ");
+            },
+            Token::LPar => print!("("),
+            Token::RPar => print!(")"),
+            Token::Bang => print!("!"),
+            Token::Slash => print!("/"),
+            Token::Pipe => print!("|"),
+            Token::Caret => print!("^"),
+            Token::Plus => print!("+"),
+            Token::Dash => print!("-"),
+            Token::Asterisk => print!("*"),
+            Token::Error(e) => print!("<error: {}>", e),
+        }
+    }
 }

src/date.rs

@@ -1,4 +1,4 @@
-use unit_defs::{DatePattern, Token};
+use ast::{DatePattern, DateToken};
 use chrono::format::Parsed;
 use std::iter::Peekable;
 use chrono::{Weekday, DateTime, UTC, FixedOffset, Duration};
@@ -10,14 +10,14 @@ pub fn parse_date<I>(
     date: &mut Peekable<I>,
     pat: &[DatePattern]
 ) -> Result<(), String>
-    where I: Iterator<Item=Token>+Clone {
+    where I: Iterator<Item=DateToken>+Clone {
 
     let tok = date.peek().cloned();
 
     macro_rules! numeric_match {
         ($name:expr, $digits:expr, $field:ident) => {
             match tok {
-                Some(Token::Number(ref s, None, None)) if $digits == 0 || s.len() == $digits => {
+                Some(DateToken::Number(ref s, None, None)) if $digits == 0 || s.len() == $digits => {
                     let value = i32::from_str_radix(&**s, 10).unwrap();
                     out.$field = Some(value as _);
                     Ok(())
@@ -31,7 +31,7 @@ pub fn parse_date<I>(
         None if tok.is_some() => Err(format!("Expected EOF, got {:?}", tok.unwrap())),
         None => return Ok(()),
         Some(&DatePattern::Literal(ref l)) => match tok {
-            Some(Token::Ident(ref s)) if s == l => Ok(()),
+            Some(DateToken::Literal(ref s)) if s == l => Ok(()),
             x => Err(format!("Expected `{}`, got {:?}", l, x)),
         },
         Some(&DatePattern::Match(ref what)) => match &**what {
@@ -47,7 +47,7 @@ pub fn parse_date<I>(
             "isoweek"   => numeric_match!("isoweek",   2, isoweek),
             "unix"      => numeric_match!("unix",      0, timestamp),
             "hour24" => match tok {
-                Some(Token::Number(ref s, None, None)) if s.len() == 2 => {
+                Some(DateToken::Number(ref s, None, None)) if s.len() == 2 => {
                     let value = u32::from_str_radix(&**s, 10).unwrap();
                     out.hour_div_12 = Some(value / 12);
                     out.hour_mod_12 = Some(value % 12);
@@ -56,12 +56,12 @@ pub fn parse_date<I>(
                 x => Err(format!("Expected 2-digit hour24, got {:?}", x))
             },
             "sec" => match tok {
-                Some(Token::Number(ref s, None, None)) if s.len() == 2 => {
+                Some(DateToken::Number(ref s, None, None)) if s.len() == 2 => {
                     let value = u32::from_str_radix(&**s, 10).unwrap();
                     out.second = Some(value);
                     Ok(())
                 },
-                Some(Token::Number(ref s, Some(ref f), None)) if s.len() == 2 => {
+                Some(DateToken::Number(ref s, Some(ref f), None)) if s.len() == 2 => {
                     let secs = u32::from_str_radix(&**s, 10).unwrap();
                     let nsecs = u32::from_str_radix(&**f, 10).unwrap() * 10u32.pow(9 - f.len() as u32);
                     out.second = Some(secs);
@@ -85,19 +85,19 @@ pub fn parse_date<I>(
                         }
                     }
                 }
-                let s = match take!(Token::Plus | Token::Minus) {
-                    Token::Plus => 1, Token::Minus => -1, _ => panic!()
+                let s = match take!(DateToken::Plus | DateToken::Dash) {
+                    DateToken::Plus => 1, DateToken::Dash => -1, _ => panic!()
                 };
-                let h = take!(Token::Number(s, None, None), s);
+                let h = take!(DateToken::Number(s, None, None), s);
                 let h = i32::from_str_radix(&*h, 10).unwrap();
-                take!(Token::Colon);
-                let m = take!(Token::Number(s, None, None), s);
+                take!(DateToken::Colon);
+                let m = take!(DateToken::Number(s, None, None), s);
                 let m = i32::from_str_radix(&*m, 10).unwrap();
                 out.offset = Some(s * (h*3600 + m*60));
                 Ok(())
             },
             "monthname" => match tok {
-                Some(Token::Ident(ref s)) => {
+                Some(DateToken::Literal(ref s)) => {
                     let res = match &*s.to_lowercase() {
                         "jan" | "january" => 1,
                         "feb" | "february" => 2,
@@ -119,7 +119,7 @@ pub fn parse_date<I>(
                 x => Err(format!("Expected month name, got {:?}", x))
             },
             "weekday" => match tok {
-                Some(Token::Ident(ref s)) => {
+                Some(DateToken::Literal(ref s)) => {
                     let res = match &*s.to_lowercase() {
                         "mon" | "monday" => Weekday::Mon,
                         "tue" | "tuesday" => Weekday::Tue,
@@ -146,11 +146,11 @@ pub fn parse_date<I>(
             Ok(())
         }
         Some(&DatePattern::Dash) => match tok {
-            Some(Token::Minus) => Ok(()),
+            Some(DateToken::Dash) => Ok(()),
             x => Err(format!("Expected `-`, got {:?}", x))
         },
         Some(&DatePattern::Colon) => match tok {
-            Some(Token::Colon) => Ok(()),
+            Some(DateToken::Colon) => Ok(()),
             x => Err(format!("Expected `:`, got {:?}", x))
         },
         Some(&DatePattern::Error(ref err)) => Err(err.clone())
@@ -162,7 +162,7 @@ pub fn parse_date<I>(
     }
 }
 
-pub fn try_decode(date: &[Token], context: &Context) -> Result<DateTime<FixedOffset>, String> {
+pub fn try_decode(date: &[DateToken], context: &Context) -> Result<DateTime<FixedOffset>, String> {
     for pat in &context.datepatterns {
         let attempt = || {
             let mut parsed = Parsed::new();

src/eval.rs

@@ -4,7 +4,7 @@ use gmp::mpq::Mpq;
 use chrono::{DateTime, FixedOffset};
 use number::{Number, Unit};
 use date;
-use unit_defs::DatePattern;
+use ast::{DatePattern, Expr, SuffixOp, Def, Defs};
 use std::ops::{Add, Div, Mul, Neg, Sub};
 use std::rc::Rc;
 use factorize::{factorize, Factors};
@@ -250,9 +250,7 @@ impl Context {
 
     /// Evaluates an expression to compute its value, *excluding* `->`
     /// conversions.
-    pub fn eval(&self, expr: &::unit_defs::Expr) -> Result<Value, String> {
-        use unit_defs::{Expr, SuffixOp};
-
+    pub fn eval(&self, expr: &Expr) -> Result<Value, String> {
         macro_rules! operator {
             ($left:ident $op:ident $opname:tt $right:ident) => {{
                 let left = try!(self.eval(&**$left));
@@ -351,9 +349,7 @@ impl Context {
         }
     }
 
-    pub fn eval_unit_name(&self, expr: &::unit_defs::Expr) -> Result<BTreeMap<String, isize>, String> {
-        use unit_defs::Expr;
-
+    pub fn eval_unit_name(&self, expr: &Expr) -> Result<BTreeMap<String, isize>, String> {
         match *expr {
             Expr::Equals(ref left, ref _right) => match **left {
                 Expr::Unit(ref name) => {
@@ -426,9 +422,7 @@ impl Context {
     }
 
     /// Evaluates an expression, include `->` conversions.
-    pub fn eval_outer(&self, expr: &::unit_defs::Expr) -> Result<String, String> {
-        use unit_defs::Expr;
-
+    pub fn eval_outer(&self, expr: &Expr) -> Result<String, String> {
         let conformance_err = |top: &Number, bottom: &Number| -> String {
             use std::io::Write;
 
@@ -608,9 +602,7 @@ impl Context {
 
     /// Takes a parsed units.txt from `unit_defs::parse()`. Prints if
     /// there are errors in the file.
-    pub fn new(defs: ::unit_defs::Defs) -> Context {
-        use unit_defs::Def;
-
+    pub fn new(defs: Defs) -> Context {
         let mut ctx = Context {
             dimensions: Vec::new(),
             units: HashMap::new(),

src/gnu_units.rs

@@ -0,0 +1,453 @@
+use std::str::Chars;
+use std::iter::Peekable;
+use std::rc::Rc;
+use ast::*;
+
+#[derive(Debug, Clone)]
+pub enum Token {
+    Eof,
+    Newline,
+    Ident(String),
+    Number(String, Option<String>, Option<String>),
+    LPar,
+    RPar,
+    Bang,
+    Slash,
+    Pipe,
+    Caret,
+    Plus,
+    Dash,
+    Asterisk,
+    Error(String),
+}
+
+#[derive(Clone)]
+pub struct TokenIterator<'a>(Peekable<Chars<'a>>);
+
+impl<'a> TokenIterator<'a> {
+    pub fn new(input: &'a str) -> TokenIterator<'a> {
+        TokenIterator(input.chars().peekable())
+    }
+}
+
+fn is_ident(c: char) -> bool {
+    match c {
+        //c if c.is_alphabetic() => true,
+        //'_' | '$' | '-' | '\'' | '"' | '%' | ',' => true,
+        ' ' | '\t' | '\n' | '(' | ')' | '/' | '|' | '^' | '+' | '*' | '\\' | '#' => false,
+        _ => true
+    }
+}
+
+impl<'a> Iterator for TokenIterator<'a> {
+    type Item = Token;
+
+    fn next(&mut self) -> Option<Token> {
+        if self.0.peek() == None {
+            return Some(Token::Eof)
+        }
+        let res = match self.0.next().unwrap() {
+            ' ' | '\t' => return self.next(),
+            '\n' => Token::Newline,
+            '!' => Token::Bang,
+            '(' => Token::LPar,
+            ')' => Token::RPar,
+            '/' => Token::Slash,
+            '|' => Token::Pipe,
+            '^' => Token::Caret,
+            '-' => Token::Dash,
+            '+' => Token::Plus,
+            '*' => Token::Asterisk,
+            '\\' => match self.0.next() {
+                Some('\n') => self.next().unwrap(),
+                Some(x) => Token::Error(format!("Invalid escape: \\{}", x)),
+                None => Token::Error(format!("Unexpected EOF")),
+            },
+            '#' => {
+                while let Some(c) = self.0.next() {
+                    match c {
+                        '\n' => break,
+                        _ => ()
+                    }
+                }
+                Token::Newline
+            },
+            x @ '0'...'9' | x @ '.' => {
+                use std::ascii::AsciiExt;
+
+                let mut integer = String::new();
+                let mut frac = None;
+                let mut exp = None;
+
+                // integer component
+                if x != '.' {
+                    integer.push(x);
+                    while let Some(c) = self.0.peek().cloned() {
+                        match c {
+                            '0'...'9' => integer.push(self.0.next().unwrap()),
+                            _ => break
+                        }
+                    }
+                } else {
+                    integer.push('0');
+                }
+                // fractional component
+                if x == '.' || Some('.') == self.0.peek().cloned() {
+                    let mut buf = String::new();
+                    if x != '.' {
+                        self.0.next();
+                    }
+                    while let Some(c) = self.0.peek().cloned() {
+                        match c {
+                            '0'...'9' => buf.push(self.0.next().unwrap()),
+                            _ => break
+                        }
+                    }
+                    if buf.len() > 0 {
+                        frac = Some(buf)
+                    }
+                }
+                // exponent
+                if let Some('e') = self.0.peek().cloned().map(|x| x.to_ascii_lowercase()) {
+                    let mut buf = String::new();
+                    self.0.next();
+                    if let Some(c) = self.0.peek().cloned() {
+                        match c {
+                            '-' => {
+                                buf.push(self.0.next().unwrap());
+                            },
+                            '+' => {
+                                self.0.next();
+                            },
+                            _ => ()
+                        }
+                    }
+                    while let Some(c) = self.0.peek().cloned() {
+                        match c {
+                            '0'...'9' => buf.push(self.0.next().unwrap()),
+                            _ => break
+                        }
+                    }
+                    if buf.len() > 0 {
+                        exp = Some(buf)
+                    }
+                }
+                Token::Number(integer, frac, exp)
+            },
+            x if is_ident(x) => {
+                let mut buf = String::new();
+                buf.push(x);
+                while let Some(c) = self.0.peek().cloned() {
+                    if is_ident(c) || c.is_numeric() {
+                        buf.push(self.0.next().unwrap());
+                    } else {
+                        break;
+                    }
+                }
+                match &*buf {
+                    _ => Token::Ident(buf)
+                }
+            },
+            x => Token::Error(format!("Unknown character: '{}'", x))
+        };
+        Some(res)
+    }
+}
+
+pub type Iter<'a> = Peekable<TokenIterator<'a>>;
+
+/*fn parse_term(mut iter: &mut Iter) -> Expr {
+    match iter.next().unwrap() {
+        Token::Ident(name) => Expr::Unit(name),
+        Token::Number(num, frac, exp) => Expr::Const(num, frac, exp),
+        Token::Plus => Expr::Plus(Box::new(parse_term(iter))),
+        Token::Minus => Expr::Neg(Box::new(parse_term(iter))),
+        // NYI: Imaginary numbers
+        Token::ImaginaryUnit => Expr::Const("0".to_owned(), None, None),
+        Token::LPar => {
+            let res = parse_expr(iter);
+            match iter.next().unwrap() {
+                Token::RPar => res,
+                x => Expr::Error(format!("Expected ), got {:?}", x))
+            }
+        },
+        Token::Hash => {
+            let mut out = vec![];
+            loop {
+                match iter.next().unwrap() {
+                    Token::Hash => break,
+                    Token::Eof | Token::Comment(_) | Token::Newline =>
+                        return Expr::Error(format!("Unterminated date literal")),
+                    x => out.push(x),
+                }
+            }
+            unimplemented!()
+            //Expr::Date(out)
+        },
+        x => Expr::Error(format!("Expected term, got {:?}", x))
+    }
+}
+
+fn parse_pow(mut iter: &mut Iter) -> Expr {
+    let left = parse_term(iter);
+    match *iter.peek().unwrap() {
+        Token::Carot => {
+            iter.next();
+            let right = parse_pow(iter);
+            Expr::Pow(Box::new(left), Box::new(right))
+        },
+        _ => left
+    }
+}
+
+fn parse_mul(mut iter: &mut Iter) -> Expr {
+    let mut terms = vec![parse_pow(iter)];
+    loop { match iter.peek().cloned().unwrap() {
+        Token::DegC | Token::DegF | Token::DegRe | Token::DegRo | Token::DegDe | Token::DegN |
+        Token::Comma | Token::Equals | Token::Plus | Token::Minus | Token::DashArrow |
+        Token::TriplePipe | Token::RPar | Token::Newline | Token::Comment(_) | Token::Eof => break,
+        Token::Slash => {
+            iter.next();
+            let right = parse_pow(iter);
+            let left = if terms.len() == 1 {
+                terms.pop().unwrap()
+            } else {
+                Expr::Mul(terms)
+            };
+            terms = vec![Expr::Frac(Box::new(left), Box::new(right))]
+        },
+        Token::Asterisk => {
+            iter.next();
+        },
+        _ => terms.push(parse_pow(iter))
+    }}
+    if terms.len() == 1 {
+        terms.pop().unwrap()
+    } else {
+        Expr::Mul(terms)
+    }
+}
+
+fn parse_suffix(mut iter: &mut Iter) -> Expr {
+    let left = parse_mul(iter);
+    match iter.peek().cloned().unwrap() {
+        Token::DegC => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Celsius, Box::new(left))
+        },
+        Token::DegF => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Fahrenheit, Box::new(left))
+        },
+        Token::DegRe => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Reaumur, Box::new(left))
+        },
+        Token::DegRo => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Romer, Box::new(left))
+        },
+        Token::DegDe => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Delisle, Box::new(left))
+        },
+        Token::DegN => {
+            iter.next();
+            Expr::Suffix(SuffixOp::Newton, Box::new(left))
+        },
+        _ => left
+    }
+}
+
+fn parse_add(mut iter: &mut Iter) -> Expr {
+    let left = parse_suffix(iter);
+    match *iter.peek().unwrap() {
+        Token::Plus => {
+            iter.next();
+            let right = parse_suffix(iter);
+            Expr::Add(Box::new(left), Box::new(right))
+        },
+        Token::Minus => {
+            iter.next();
+            let right = parse_suffix(iter);
+            Expr::Sub(Box::new(left), Box::new(right))
+        },
+        _ => left
+    }
+}
+
+fn parse_eq(mut iter: &mut Iter) -> Expr {
+    let left = parse_add(iter);
+    match iter.peek().cloned().unwrap() {
+        Token::Equals => {
+            iter.next();
+            let right = parse_add(iter);
+            Expr::Equals(Box::new(left), Box::new(right))
+        },
+        _ => left
+    }
+}
+
+pub fn parse_expr(mut iter: &mut Iter) -> Expr {
+    match iter.peek().cloned() {
+        Some(Token::Ident(ref s)) if s == "factorize" => {
+            iter.next();
+            return Expr::Factorize(Box::new(parse_eq(iter)))
+        },
+        _ => ()
+    }
+    let left = parse_eq(iter);
+    match iter.peek().cloned().unwrap() {
+        Token::DashArrow => {
+            iter.next();
+            let right = match iter.peek().cloned().unwrap() {
+                Token::DegC => Expr::DegC,
+                Token::DegF => Expr::DegF,
+                Token::DegRe => Expr::DegRe,
+                Token::DegRo => Expr::DegRo,
+                Token::DegDe => Expr::DegDe,
+                Token::DegN => Expr::DegN,
+                _ => parse_eq(iter)
+            };
+            Expr::Convert(Box::new(left), Box::new(right))
+        },
+        _ => left
+    }
+}
+
+fn parse_unknown(mut iter: &mut Iter) {
+    loop {
+        match iter.peek().cloned().unwrap() {
+            Token::Newline | Token::Comment(_) | Token::Eof => break,
+            _ => iter.next()
+        };
+    }
+}
+
+fn parse_alias(mut iter: &mut Iter) -> Option<(Expr, String)> {
+    match *iter.peek().unwrap() {
+        Token::Newline | Token::Comment(_) | Token::Eof => return None,
+        _ => ()
+    }
+    let expr = parse_expr(iter);
+    match iter.next().unwrap() {
+        Token::TriplePipe => (),
+        _ => return None
+    };
+    let name = match iter.next().unwrap() {
+        Token::Ident(name) => name,
+        _ => return None
+    };
+    match iter.peek().cloned().unwrap() {
+        Token::Newline | Token::Comment(_) | Token::Eof => (),
+        _ => return None
+    };
+    Some((expr, name))
+}
+
+fn parse_datepattern(mut iter: &mut Iter) -> Vec<DatePattern> {
+    let mut out = vec![];
+    loop {
+        match iter.peek().cloned().unwrap() {
+            Token::Newline | Token::Comment(_) | Token::Eof | Token::RBrack => break,
+            Token::Ident(name) => out.push(DatePattern::Match(name)),
+            Token::Quote(name) => out.push(DatePattern::Literal(name)),
+            Token::Minus => out.push(DatePattern::Dash),
+            Token::Colon => out.push(DatePattern::Colon),
+            Token::LBrack => {
+                iter.next();
+                let res = DatePattern::Optional(parse_datepattern(iter));
+                let res = match iter.next().unwrap() {
+                    Token::RBrack => res,
+                    x => DatePattern::Error(format!("Expected ], got {:?}", x))
+                };
+                out.push(res)
+            },
+            x => out.push(DatePattern::Error(format!("Unexpected token {:?} in date pattern", x)))
+        }
+        iter.next();
+    }
+    out
+}
+
+pub fn parse(mut iter: &mut Iter) -> Defs {
+    let mut map = vec![];
+    let mut aliases = vec![];
+    let mut line = 1;
+    loop {
+        let mut copy = iter.clone();
+        if let Some(a) = parse_alias(&mut copy) {
+            aliases.push(a);
+            *iter = copy;
+            continue
+        }
+        match iter.next().unwrap() {
+            Token::Newline => line += 1,
+            Token::Comment(lines) => line += lines,
+            Token::Eof => break,
+            Token::Ident(ref name) if name == "datepattern" =>
+                map.push((name.clone(), Rc::new(Def::DatePattern(parse_datepattern(iter))))),
+            Token::Ident(name) => {
+                let def = match iter.next().unwrap() {
+                    Token::ColonDash => {
+                        let expr = parse_expr(iter);
+                        Def::Prefix(expr)
+                    },
+                    Token::DColonDash => {
+                        let expr = parse_expr(iter);
+                        Def::SPrefix(expr)
+                    },
+                    Token::EqBangEq => match iter.next().unwrap() {
+                        Token::Ident(val) => Def::Dimension(val),
+                        _ => Def::Error(format!("Line {}: Malformed dimensionless unit", line))
+                    },
+                    Token::ColonEq => Def::Unit(parse_expr(iter)),
+                    Token::LBrack => {
+                        // NYI
+                        let mut n = 0;
+                        let mut first = true;
+                        loop {
+                            match iter.peek().cloned().unwrap() {
+                                Token::LBrace => n += 1,
+                                Token::RBrace if n == 1 => {
+                                    iter.next();
+                                    break
+                                },
+                                Token::RBrace => n -= 1,
+                                Token::Newline | Token::Comment(_) if !first && n == 0 => break,
+                                Token::Newline if first => first = false,
+                                Token::Eof => break,
+                                Token::Comment(lines) => line += lines,
+                                _ => ()
+                            }
+                            iter.next();
+                        }
+                        continue
+                        //Def::Error(format!("NYI: functions"))
+                    }
+                    _ => {
+                        parse_unknown(iter);
+                        Def::Error(format!("Line {}: Unknown definition", line))
+                    }
+                };
+                map.push((name.clone(), Rc::new(def)));
+            },
+            _ => parse_unknown(iter)
+        };
+    }
+    Defs {
+        defs: map,
+        aliases: aliases,
+    }
+}
+*/
+pub fn tokens(mut iter: &mut Iter) -> Vec<Token> {
+    let mut out = vec![];
+    loop {
+        match iter.next().unwrap() {
+            Token::Eof => break,
+            x => out.push(x)
+        }
+    }
+    out
+}

src/lib.rs

@@ -40,6 +40,8 @@ pub mod eval;
 pub mod number;
 pub mod date;
 pub mod factorize;
+pub mod gnu_units;
+pub mod ast;
 
 pub use number::Number;
 pub use eval::{Context, Value};

src/unit_defs.rs

@@ -1,6 +1,7 @@
 use std::str::Chars;
 use std::iter::Peekable;
 use std::rc::Rc;
+use ast::*;
 
 #[derive(Debug, Clone)]
 pub enum Token {
@@ -274,69 +275,6 @@ impl<'a> Iterator for TokenIterator<'a> {
 
 pub type Iter<'a> = Peekable<TokenIterator<'a>>;
 
-#[derive(Debug, Clone)]
-pub enum SuffixOp {
-    Celsius,
-    Fahrenheit,
-    Reaumur,
-    Romer,
-    Delisle,
-    Newton,
-}
-
-#[derive(Debug, Clone)]
-pub enum Expr {
-    Unit(String),
-    Quote(String),
-    Const(String, Option<String>, Option<String>),
-    Date(Vec<Token>),
-    Frac(Box<Expr>, Box<Expr>),
-    Mul(Vec<Expr>),
-    Pow(Box<Expr>, Box<Expr>),
-    Add(Box<Expr>, Box<Expr>),
-    Sub(Box<Expr>, Box<Expr>),
-    Neg(Box<Expr>),
-    Plus(Box<Expr>),
-    Convert(Box<Expr>, Box<Expr>),
-    Equals(Box<Expr>, Box<Expr>),
-    Suffix(SuffixOp, Box<Expr>),
-    Call(String, Vec<Expr>),
-    Factorize(Box<Expr>),
-    DegC,
-    DegF,
-    DegRe,
-    DegRo,
-    DegDe,
-    DegN,
-    Error(String),
-}
-
-#[derive(Debug, Clone)]
-pub enum DatePattern {
-    Literal(String),
-    Match(String),
-    Optional(Vec<DatePattern>),
-    Dash,
-    Colon,
-    Error(String),
-}
-
-#[derive(Debug)]
-pub enum Def {
-    Dimension(String),
-    Prefix(Expr),
-    SPrefix(Expr),
-    Unit(Expr),
-    DatePattern(Vec<DatePattern>),
-    Error(String),
-}
-
-#[derive(Debug)]
-pub struct Defs {
-    pub defs: Vec<(String, Rc<Def>)>,
-    pub aliases: Vec<(Expr, String)>,
-}
-
 fn is_func(name: &str) -> bool {
     match name {
         "sqrt" => true,
@@ -394,7 +332,8 @@ fn parse_term(mut iter: &mut Iter) -> Expr {
                     x => out.push(x),
                 }
             }
-            Expr::Date(out)
+            unimplemented!()
+            //Expr::Date(out)
         },
         x => Expr::Error(format!("Expected term, got {:?}", x))
     }