Merge pull request #425 from Nadrieril/run-on-popular-crates

charon/Cargo.toml

@@ -45,9 +45,11 @@ assert_cmd = "2.0"
 clap = { version = "4.0", features = ["derive", "env"] }
 colored = "2.0.4"
 convert_case = "0.6.0"
+crates_io_api = { version = "0.11.0", optional = true }
 derivative = "2.2.0"
 derive-visitor = { version = "0.4.0", features = ["std-types-drive"] }
 env_logger = { version = "0.11", features = ["color"] }
+flate2 = { version = "1.0.34", optional = true }
 hashlink = { version = "0.9", features = ["serde_impl"] }
 im = "15.1.0"
 index_vec = { version = "0.1.3", features = ["serde"] }
@@ -60,6 +62,7 @@ nom-supreme = "0.8.0"
 petgraph = "0.6.2"
 pretty = "0.12"
 regex = "1.7.1"
+reqwest = { version = "0.12.8", optional = true }
 rustc_apfloat = "0.2.1"
 rustc_version = "0.4"
 serde_json = { version = "1.0.91", features = ["unbounded_depth"] }
@@ -68,11 +71,12 @@ serde_stacker = "0.1.11"
 serde = { version = "1.0.152", features = ["derive", "rc"] }
 stacker = "0.1"
 take_mut = "0.2.2"
+tar = { version = "0.4.42", optional = true }
 toml = { version = "0.8", features = ["parse"] }
 tracing-subscriber = { version = "0.3", features = [ "env-filter", "std", "fmt" ] }
 tracing-tree = { git = "https://github.com/Nadrieril/tracing-tree", features = [ "time" ] } # Fork with improved formating and timing info.
-# tracing-tree = { path = "../../tracing-tree", features = [ "time" ] }
 tracing = { version = "0.1", features = [ "max_level_trace" ] }
+wait-timeout = { version = "0.2.0", optional = true }
 which = "6.0.1"
 
 hax-frontend-exporter = { git = "https://github.com/hacspec/hax", branch = "main", optional = true }
@@ -85,6 +89,15 @@ default = ["rustc"]
 # disabled, the binaries won't build but the main crate (with the ast
 # definitions) still does. It is enabled by default.
 rustc = ["dep:hax-frontend-exporter"]
+# This feature enables the `popular-crates` test which runs Charon on the most downloaded crates from crates.io.
+popular-crates-test = [
+    "dep:crates_io_api",
+    "dep:flate2",
+    "dep:reqwest",
+    "dep:tar",
+    "dep:wait-timeout",
+]
+wait-timeout = ["dep:wait-timeout"]
 
 [dev-dependencies]
 ignore = "0.4"

charon/Makefile

@@ -13,6 +13,10 @@ format:
 test:
 	cargo test
 
+.PHONY: test-popular-crates
+test-popular-crates:
+	cargo test --release --features popular-crates-test --test popular-crates -- --test-threads 8
+
 # Build the doc.
 # For some reason, I don't manage to build all the packages in one command.
 .PHONY: doc

charon/src/bin/charon-driver/main.rs

@@ -276,6 +276,13 @@ fn main() {
         }
     }
 
+    if !errors_as_warnings && matches!(res, Err(CharonFailure::Panic)) {
+        // If we emitted any error, the call into rustc will panic. Hence we assume this is
+        // just a normal failure.
+        // TODO: emit errors ourselves to avoid this (#409).
+        res = Err(CharonFailure::RustcError(error_count));
+    }
+
     match res {
         Ok(()) => {
             if error_count > 0 {
@@ -284,16 +291,14 @@ fn main() {
                 log::warn!("{}", msg);
             }
         }
-        Err(err) if errors_as_warnings => {
-            log::error!("{err}");
-        }
-        Err(CharonFailure::Panic) => {
-            // Standard rust panic error code.
-            std::process::exit(101);
-        }
-        Err(err @ (CharonFailure::RustcError(_) | CharonFailure::Serialize)) => {
+        Err(err) => {
             log::error!("{err}");
-            std::process::exit(1);
+            if matches!(err, CharonFailure::Panic) {
+                // This is a real panic, exit with the standard rust panic error code.
+                std::process::exit(101);
+            } else if !errors_as_warnings {
+                std::process::exit(1);
+            }
         }
     }
 }

charon/src/bin/charon/main.rs

@@ -248,6 +248,7 @@ pub fn main() -> anyhow::Result<()> {
         Ok(())
     } else {
         let code = exit_status.code().unwrap_or(-1);
-        bail!("Charon driver exited with code {code}")
+        // Rethrow the exit code
+        std::process::exit(code);
     }
 }

charon/tests/popular-crates.rs

@@ -0,0 +1,164 @@
+//! This test downlads the `NUMBER_OF_CRATES` most downloaded crates from crates.io and runs charon
+//! on each of them.
+//!
+//! This test requires a feature flag. To run, call `make test-popular-crates`.
+#![feature(path_add_extension)]
+#![cfg(feature = "popular-crates-test")]
+use anyhow::{bail, Context, Result};
+use assert_cmd::prelude::CommandCargoExt;
+use crates_io_api::Version;
+use flate2::read::GzDecoder;
+use itertools::Itertools;
+use std::{
+    fs::File,
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+    sync::Arc,
+    time::Duration,
+};
+use tar::Archive;
+use wait_timeout::ChildExt;
+
+static TESTS_DIR: &str = "tests/popular-crates";
+
+const NUMBER_OF_CRATES: u64 = 50;
+
+/// List of crates that we know we can't extract.
+static BLACKLIST: &[&str] = &[
+    "aho-corasick",
+    "base64",
+    "bitflags",
+    "cc",
+    "clap",
+    "crossbeam-utils",
+    "getrandom",
+    "idna",
+    "itertools",
+    "num-traits",
+    "parking_lot",
+    "parking_lot_core",
+    "proc-macro2",
+    "rand",
+    "regex-automata",
+    "regex-syntax",
+    "semver",
+    "serde",
+    "serde_json",
+    "syn",
+    "time",
+];
+
+/// Downloads and extracts the crate into a subdirectory of `TESTS_DIR` and returns the path to
+/// that directory.
+fn extract_crate(version: &Version) -> Result<PathBuf> {
+    let full_name = &format!("{}-{}", version.crate_name, version.num);
+    let download_url = format!("https://crates.io{}", version.dl_path);
+    let directory = PathBuf::from(format!("{}/{}", TESTS_DIR, full_name));
+    if directory.exists() {
+        // Assuùe the directory already contains the extracted crate.
+        return Ok(directory);
+    }
+
+    let archive_path = {
+        let mut path = directory.clone();
+        path.add_extension("tar.gz");
+        path
+    };
+    {
+        // Download the crate archive
+        let mut temp_file = File::create(&archive_path)
+            .with_context(|| format!("while creating `{}`", archive_path.display()))?;
+        reqwest::blocking::get(download_url)?.copy_to(&mut temp_file)?;
+    }
+    {
+        // Extract the crate archive
+        let temp_file = File::open(&archive_path)?;
+        let mut archive = Archive::new(GzDecoder::new(temp_file));
+        // This assumes that the archive always contains exactly one folder named
+        // `{crate_name}-{version}`, which seems to be the case. Worst case we get unexpected files
+        // inside the `popular-crates` subfolder.
+        archive
+            .unpack(TESTS_DIR)
+            .with_context(|| "while extracting archive")?;
+    }
+    std::fs::remove_file(archive_path)?;
+
+    Ok(directory)
+}
+
+fn process_crate(version: &Version) -> Result<()> {
+    let crate_dir = extract_crate(version)?;
+    let llbc_path = {
+        // Relative to the crate directory
+        let mut path = Path::new("..").to_path_buf();
+        path.push(crate_dir.file_name().unwrap());
+        path.add_extension("llbc");
+        path
+    };
+
+    // Call charon
+    let mut child = Command::cargo_bin("charon")?
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .current_dir(&crate_dir)
+        .arg("--hide-marker-traits")
+        .arg("--errors-as-warnings")
+        .arg("--dest-file")
+        .arg(&llbc_path)
+        .spawn()?;
+    let timeout = Duration::from_secs(30);
+    let status_code = match child.wait_timeout(timeout)? {
+        Some(status) => status,
+        None => {
+            child.kill()?;
+            eprintln!("Compilation timed out");
+            child.wait()?
+        }
+    };
+    // let output = cmd.output()?;
+
+    if !status_code.success() {
+        let stderr = std::io::read_to_string(child.stderr.unwrap())?;
+        bail!("Compilation failed: {stderr}")
+    }
+
+    Ok(())
+}
+
+#[test] // Only include in release mode
+pub fn test_popular_crates() -> Result<()> {
+    use crates_io_api::*;
+    let client = Arc::new(
+        SyncClient::new(
+            "charon-test-runner (Nadrieril@users.noreply.github.com)",
+            std::time::Duration::from_millis(1000),
+        )
+        .unwrap(),
+    );
+
+    let q = CratesQuery::builder()
+        .sort(Sort::Downloads)
+        .page_size(NUMBER_OF_CRATES)
+        .build();
+    let crates = client.crates(q)?;
+
+    let tests: Vec<_> = crates
+        .crates
+        .into_iter()
+        .map(|krate| {
+            let known_failure = BLACKLIST.contains(&krate.name.as_str());
+            let name = format!("{}-{}", krate.name, krate.max_version);
+            let client = Arc::clone(&client);
+            let test = libtest_mimic::Trial::test(name, move || {
+                let krate = client.get_crate(&krate.name)?;
+                let version = krate.versions.into_iter().next().unwrap();
+                process_crate(&version).map_err(|err| err.into())
+            })
+            .with_ignored_flag(known_failure);
+            Ok::<_, Error>(test)
+        })
+        .try_collect()?;
+
+    let args = libtest_mimic::Arguments::from_args();
+    libtest_mimic::run(&args, tests).exit()
+}

charon/tests/popular-crates/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

charon/tests/ui/error-dependencies.out

@@ -18,4 +18,4 @@ error: The external definition `core::ptr::metadata::Thin` triggered errors. It
 
 error: aborting due to 3 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 2 errors

charon/tests/ui/generic-associated-types.out

@@ -80,4 +80,4 @@ error: Error during trait resolution: Found unsupported GAT `Type` when resolvin
 
 error: aborting due to 12 previous errors; 1 warning emitted
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 12 errors

charon/tests/ui/issue-378-ctor-as-fn.out

@@ -15,4 +15,4 @@ error: The external definition `core::option::Option::Some` triggered errors. It
 
 error: aborting due to 3 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 2 errors

charon/tests/ui/issue-393-shallowinitbox.out

@@ -8,4 +8,4 @@ error: Unexpected `ShallowInitBox`
 
 error: aborting due to 1 previous error
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 1 errors

charon/tests/ui/non-lifetime-gats.out

@@ -48,4 +48,4 @@ error: Ignoring the following item due to an error: test_crate::moar_variables::
 
 error: aborting due to 8 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 8 errors

charon/tests/ui/rename_attribute_failure.out

@@ -36,4 +36,4 @@ error: Error parsing attribute: Unrecognized attribute: `charon::something_else(
 
 error: aborting due to 6 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 6 errors

charon/tests/ui/unsupported/advanced-const-generics.out

@@ -34,4 +34,4 @@ error: Ignoring the following item due to an error: test_crate::foo
 
 error: aborting due to 3 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 2 errors

charon/tests/ui/unsupported/unbound-lifetime.out

@@ -27,4 +27,4 @@ error: Ignoring the following item due to an error: test_crate::get
 error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0261`.
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 2 errors

charon/tests/ui/unsupported/well-formedness-bound.out

@@ -14,4 +14,4 @@ error: Ignoring the following item due to an error: test_crate::get
 
 error: aborting due to 2 previous errors
 
-Error: Charon driver exited with code 101
+ERROR Compilation encountered 2 errors