Wire up yaml-test-suite

Cargo.toml

@@ -11,3 +11,8 @@ harness = false
 [[test]]
 name = "test_version"
 harness = false
+
+[workspace]
+[dev-dependencies]
+pretty_assertions = "1.0"
+unsafe-libyaml-test-suite = { path = "tests/data" }

tests/data/.gitignore

@@ -0,0 +1 @@
+/yaml-test-suite

tests/data/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "unsafe-libyaml-test-suite"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[lib]
+path = "lib.rs"
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+
+[build-dependencies]
+anyhow = "1.0"
+flate2 = "1.0"
+reqwest = { version = "0.11", features = ["blocking"] }
+tar = "0.4.16"

tests/data/build.rs

@@ -0,0 +1,44 @@
+use anyhow::Result;
+use flate2::read::GzDecoder;
+use std::fs;
+use std::path::Path;
+use tar::Archive;
+
+const TAG: &str = "data-2020-02-11";
+
+fn main() {
+    let needs_clone = match fs::read_to_string("yaml-test-suite/COMMIT") {
+        Err(_) => true,
+        Ok(contents) => contents.trim() != TAG,
+    };
+    if needs_clone {
+        download_and_unpack().unwrap();
+    }
+}
+
+fn download_and_unpack() -> Result<()> {
+    let url = format!("https://github.com/yaml/yaml-test-suite/archive/refs/tags/{TAG}.tar.gz");
+    let response = reqwest::blocking::get(&url)?.error_for_status()?;
+    let decoder = GzDecoder::new(response);
+    let mut archive = Archive::new(decoder);
+    let prefix = format!("yaml-test-suite-{}", TAG);
+
+    let yaml_test_suite = Path::new("yaml-test-suite");
+    if yaml_test_suite.exists() {
+        fs::remove_dir_all(yaml_test_suite)?;
+    }
+
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let path = entry.path()?;
+        if path == Path::new("pax_global_header") {
+            continue;
+        }
+        let relative = path.strip_prefix(&prefix)?;
+        let out = yaml_test_suite.join(relative);
+        entry.unpack(&out)?;
+    }
+
+    fs::write("yaml-test-suite/COMMIT", TAG)?;
+    Ok(())
+}

tests/data/lib.rs

@@ -0,0 +1,79 @@
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+use std::collections::BTreeSet as Set;
+use std::fs::{self, File};
+use std::io::{BufRead, BufReader};
+use std::path::Path;
+
+#[proc_macro]
+pub fn test_emitter(_input: TokenStream) -> TokenStream {
+    test("libyaml-emitter", |dir| !dir.join("error").exists())
+}
+
+#[proc_macro]
+pub fn test_parser(_input: TokenStream) -> TokenStream {
+    test("libyaml-parser", |dir| !dir.join("error").exists())
+}
+
+#[proc_macro]
+pub fn test_parser_error(_input: TokenStream) -> TokenStream {
+    test("libyaml-parser-error", |dir| dir.join("error").exists())
+}
+
+fn test(ignorelist: &str, check: fn(&Path) -> bool) -> TokenStream {
+    let tests_dir = Path::new("tests");
+
+    let mut ignored_ids = Set::new();
+    let ignorelist = tests_dir.join("ignorelist").join(ignorelist);
+    for line in BufReader::new(File::open(ignorelist).unwrap()).lines() {
+        let mut line = line.unwrap();
+        line.truncate(4);
+        ignored_ids.insert(line);
+    }
+
+    let mut tests = proc_macro2::TokenStream::new();
+    let ignore = quote!(#[ignore]);
+    let yaml_test_suite = tests_dir.join("data").join("yaml-test-suite");
+    for entry in fs::read_dir(yaml_test_suite).unwrap() {
+        let entry = entry.unwrap();
+        if !entry.file_type().unwrap().is_dir() {
+            continue;
+        }
+
+        let path = entry.path();
+        let description = path.join("===");
+        let slug = if let Ok(description) = fs::read_to_string(description) {
+            description_to_slug(description)
+        } else {
+            continue;
+        };
+
+        if !check(&path) {
+            continue;
+        }
+
+        let file_name = entry.file_name();
+        let id = file_name.to_str().unwrap();
+        let test_name = format_ident!("_{id}_{slug}");
+        let ignore = ignored_ids.contains(id).then_some(&ignore);
+
+        tests.extend(quote! {
+            #[test]
+            #ignore
+            #[allow(non_snake_case)]
+            fn #test_name() {
+                test(#id);
+            }
+        });
+    }
+
+    TokenStream::from(tests)
+}
+
+fn description_to_slug(mut description: String) -> String {
+    description = description.replace(|ch: char| !ch.is_ascii_alphanumeric(), "_");
+    while description.contains("__") {
+        description = description.replace("__", "_");
+    }
+    description.trim_matches('_').to_ascii_lowercase()
+}

tests/ignorelist/libyaml-emitter

@@ -0,0 +1,72 @@
+26DV: Whitespace around colon in mappings
+2EBW: Allowed characters in keys
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+2XXW: Spec Example 2.25. Unordered Sets
+3MYT: Plain Scalar looking like key, comment, anchor and tag
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3]
+52DL: Explicit Non-Specific Tag [1.3]
+565N: Construct Binary
+5TYM: Spec Example 6.21. Local Tag Prefix
+5WE3: Spec Example 8.17. Explicit Block Mapping Entries
+6CK3: Spec Example 6.26. Tag Shorthands
+6FWR: Block Scalar Keep
+6KGN: Anchor for empty node
+6M2F: Aliases in Explicit Block Mapping
+6PBE: Zero-indented sequences in explicit mapping keys
+6SLA: Allowed characters in quoted mapping key
+6WLZ: Spec Example 6.18. Primary Tag Handle [1.3]
+6WPF: Spec Example 6.8. Flow Folding [1.3]
+6XDY: Two document start markers
+6ZKB: Spec Example 9.6. Stream
+7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines
+7W2P: Block Mapping with Missing Values
+7Z25: Bare document after document end marker
+8KB6: Multiline plain flow mapping key without value
+8XYN: Anchor with unicode character
+9BXH: Multiline doublequoted flow mapping key without value
+8MK2: Explicit Non-Specific Tag
+9DXL: Spec Example 9.6. Stream [1.3]
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9TFX: Spec Example 7.6. Double Quoted Lines [1.3]
+B3HG: Spec Example 8.9. Folded Scalar [1.3]
+C2DT: Spec Example 7.18. Flow Mapping Adjacent Values
+DFF7: Spec Example 7.16. Flow Mapping Entries
+E76Z: Aliases in Implicit Block Mapping
+EX5H: Multiline Scalar at Top Level [1.3]
+EXG3: Three dashes and content without space [1.3]
+FBC9: Allowed characters in plain scalars
+FH7J: Tags on Empty Scalars
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+J3BT: Spec Example 5.12. Tabs and Spaces
+JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3]
+JTV5: Block Mapping with Multiline Scalars
+K54U: Tab after document header
+KK5P: Various combinations of explicit block mappings
+KSS4: Scalars on --- line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+LE5A: Spec Example 7.24. Flow Nodes
+M7A3: Spec Example 9.3. Bare Documents
+M9B4: Spec Example 8.7. Literal Scalar
+NAT4: Various empty or newline only quoted strings
+NHX8: Empty Lines at End of Document
+PUW8: Document start on last line
+PW8X: Anchors on Empty Scalars
+Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3]
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+S4JQ: Spec Example 6.28. Non-Specific Tags
+T26H: Spec Example 8.8. Literal Content [1.3]
+T4YY: Spec Example 7.9. Single Quoted Lines [1.3]
+T5N4: Spec Example 8.7. Literal Scalar [1.3]
+UT92: Spec Example 9.4. Explicit Documents
+W42U: Spec Example 8.15. Block Sequence Entry Types
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+X38W: Aliases in Flow Objects
+XLQ9: Multiline scalar that looks like a YAML directive
+Y2GN: Anchor with colon in the middle
+ZWK4: Key with anchor after missing explicit mapping value

tests/ignorelist/libyaml-parser

@@ -0,0 +1,34 @@
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+5MUD: Colon and adjacent value on next line
+6BCT: Spec Example 6.3. Separation Spaces
+6LVF: Spec Example 6.13. Reserved Directives
+6M2F: Aliases in Explicit Block Mapping
+7Z25: Bare document after document end marker
+8XYN: Anchor with unicode character
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9SA2: Multiline double quoted flow mapping key
+A2M4: Spec Example 6.2. Indentation Indicators
+BEC7: Spec Example 6.14. “YAML” directive
+DBG4: Spec Example 7.10. Plain Characters
+DK3J: Zero indented block scalar with line that looks like a comment
+FP8R: Zero indented block scalar
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+HWV9: Document-end marker
+K3WX: Colon and adjacent value after comment on next line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+M7A3: Spec Example 9.3. Bare Documents
+NHX8: Empty Lines at End of Document
+NJ66: Multiline plain flow mapping key
+Q5MG: Tab at beginning of line followed by a flow mapping
+QT73: Comment and document-end marker
+R4YG: Spec Example 8.2. Block Indentation Indicator
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+UT92: Spec Example 9.4. Explicit Documents
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+Y2GN: Anchor with colon in the middle

tests/ignorelist/libyaml-parser-error

@@ -0,0 +1,10 @@
+9C9N: Wrong indented flow sequence
+9HCY: Need document footer before directives
+9JBA: Invalid comment after end of flow sequence
+CVW2: Invalid comment after comma
+EB22: Missing document-end marker before directive
+QB6E: Wrong indented multiline quoted scalar
+RHX7: YAML directive without document end marker
+S98Z: Block scalar with more spaces than first content line
+SU5Z: Comment without whitespace after doublequoted scalar
+X4QW: Comment without whitespace after block scalar indicator

tests/test_emitter.rs

@@ -0,0 +1,31 @@
+use std::fs;
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+fn test(id: &str) {
+    let dir = Path::new("tests")
+        .join("data")
+        .join("yaml-test-suite")
+        .join(id);
+
+    let output = Command::new(env!("CARGO_BIN_EXE_run_emitter_test_suite"))
+        .arg(dir.join("test.event"))
+        .stdin(Stdio::null())
+        .output()
+        .unwrap();
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    eprint!("{}", stderr);
+
+    let out = if dir.join("out.yaml").exists() {
+        dir.join("out.yaml")
+    } else {
+        dir.join("in.yaml")
+    };
+    let expected = fs::read_to_string(out).unwrap();
+    pretty_assertions::assert_str_eq!(expected, stdout);
+    assert!(output.status.success());
+}
+
+unsafe_libyaml_test_suite::test_emitter!();

tests/test_parser.rs

@@ -0,0 +1,26 @@
+use std::fs;
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+fn test(id: &str) {
+    let dir = Path::new("tests")
+        .join("data")
+        .join("yaml-test-suite")
+        .join(id);
+
+    let output = Command::new(env!("CARGO_BIN_EXE_run_parser_test_suite"))
+        .arg(dir.join("in.yaml"))
+        .stdin(Stdio::null())
+        .output()
+        .unwrap();
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    eprint!("{}", stderr);
+
+    let expected = fs::read_to_string(dir.join("test.event")).unwrap();
+    pretty_assertions::assert_str_eq!(expected, stdout);
+    assert!(output.status.success());
+}
+
+unsafe_libyaml_test_suite::test_parser!();

tests/test_parser_error.rs

@@ -0,0 +1,25 @@
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+fn test(id: &str) {
+    let dir = Path::new("tests")
+        .join("data")
+        .join("yaml-test-suite")
+        .join(id);
+
+    let output = Command::new(env!("CARGO_BIN_EXE_run_parser_test_suite"))
+        .arg(dir.join("in.yaml"))
+        .stdin(Stdio::null())
+        .output()
+        .unwrap();
+
+    if output.status.success() {
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        eprint!("{}", stdout);
+        eprint!("{}", stderr);
+        panic!("expected parse to fail");
+    }
+}
+
+unsafe_libyaml_test_suite::test_parser_error!();