Skip to content

Commit

Permalink
Auto merge of #12939 - Alexendoo:lintcheck-popular-crates, r=xFrednet
Browse files Browse the repository at this point in the history
Merge lintcheck popular-crates bin as a subcommand

Also rewrites it to use `ureq` to drop some heavy dependencies as `crates_io_api` brings in `reqwest`

r? `@xFrednet`

changelog: none
  • Loading branch information
bors committed Jun 16, 2024
2 parents a2c9782 + 3a983c3 commit 8065e0f
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 88 deletions.
13 changes: 1 addition & 12 deletions lintcheck/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,19 @@ publish = false
default-run = "lintcheck"

[dependencies]
anyhow = "1.0.69"
cargo_metadata = "0.15.3"
clap = { version = "4.4", features = ["derive", "env"] }
crates_io_api = "0.8.1"
crossbeam-channel = "0.5.6"
diff = "0.1.13"
flate2 = "1.0"
indicatif = "0.17.3"
rayon = "1.5.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.85"
strip-ansi-escapes = "0.1.1"
tar = "0.4"
toml = "0.7.3"
ureq = "2.2"
ureq = { version = "2.2", features = ["json"] }
walkdir = "2.3"

[features]
deny-warnings = []

[[bin]]
name = "lintcheck"
path = "src/main.rs"

[[bin]]
name = "popular-crates"
path = "src/popular-crates.rs"
6 changes: 3 additions & 3 deletions lintcheck/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ the repo root.
The results will then be saved to `lintcheck-logs/custom_logs.toml`.

The `custom.toml` file may be built using <https://crates.io> recently most
downloaded crates by using the `popular-crates` binary from the `lintcheck`
directory. For example, to retrieve the 100 recently most downloaded crates:
downloaded crates by using `cargo lintcheck popular`. For example, to retrieve
the 200 recently most downloaded crates:

```
cargo run --release --bin popular-crates -- -n 100 custom.toml
cargo lintcheck popular -n 200 custom.toml
```


Expand Down
9 changes: 9 additions & 0 deletions lintcheck/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,16 @@ pub(crate) struct LintcheckConfig {

#[derive(Subcommand, Clone, Debug)]
pub(crate) enum Commands {
/// Display a markdown diff between two lintcheck log files in JSON format
Diff { old: PathBuf, new: PathBuf },
/// Create a lintcheck crates TOML file containing the top N popular crates
Popular {
/// Output TOML file name
output: PathBuf,
/// Number of crate names to download
#[clap(short, long, default_value_t = 100)]
number: usize,
},
}

#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)]
Expand Down
20 changes: 12 additions & 8 deletions lintcheck/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
mod config;
mod driver;
mod json;
mod popular_crates;
mod recursive;

use crate::config::{Commands, LintcheckConfig, OutputFormat};
Expand All @@ -43,21 +44,21 @@ const LINTCHECK_DOWNLOADS: &str = "target/lintcheck/downloads";
const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";

/// List of sources to check, loaded from a .toml file
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Deserialize)]
struct SourceList {
crates: HashMap<String, TomlCrate>,
#[serde(default)]
recursive: RecursiveOptions,
}

#[derive(Debug, Serialize, Deserialize, Default)]
#[derive(Debug, Deserialize, Default)]
struct RecursiveOptions {
ignore: HashSet<String>,
}

/// A crate source stored inside the .toml
/// will be translated into on one of the `CrateSource` variants
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Deserialize)]
struct TomlCrate {
name: String,
versions: Option<Vec<String>>,
Expand All @@ -69,7 +70,7 @@ struct TomlCrate {

/// Represents an archive we download from crates.io, or a git repo, or a local repo/folder
/// Once processed (downloaded/extracted/cloned/copied...), this will be translated into a `Crate`
#[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)]
#[derive(Debug, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)]
enum CrateSource {
CratesIo {
name: String,
Expand Down Expand Up @@ -609,7 +610,6 @@ fn gather_stats(warnings: &[ClippyWarning]) -> (String, HashMap<&String, usize>)
(stats_string, counter)
}

#[allow(clippy::too_many_lines)]
fn main() {
// We're being executed as a `RUSTC_WRAPPER` as part of `--recursive`
if let Ok(addr) = env::var("LINTCHECK_SERVER") {
Expand All @@ -624,11 +624,15 @@ fn main() {

let config = LintcheckConfig::new();

if let Some(Commands::Diff { old, new }) = config.subcommand {
json::diff(&old, &new);
return;
match config.subcommand {
Some(Commands::Diff { old, new }) => json::diff(&old, &new),
Some(Commands::Popular { output, number }) => popular_crates::fetch(output, number).unwrap(),
None => lintcheck(config),
}
}

#[allow(clippy::too_many_lines)]
fn lintcheck(config: LintcheckConfig) {
println!("Compiling clippy...");
build_clippy();
println!("Done compiling");
Expand Down
65 changes: 0 additions & 65 deletions lintcheck/src/popular-crates.rs

This file was deleted.

52 changes: 52 additions & 0 deletions lintcheck/src/popular_crates.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
use serde::Deserialize;
use std::error::Error;
use std::fmt::Write;
use std::fs;
use std::path::PathBuf;

#[derive(Deserialize, Debug)]
struct Page {
crates: Vec<Crate>,
meta: Meta,
}

#[derive(Deserialize, Debug)]
struct Crate {
name: String,
max_version: String,
}

#[derive(Deserialize, Debug)]
struct Meta {
next_page: String,
}

pub(crate) fn fetch(output: PathBuf, number: usize) -> Result<(), Box<dyn Error>> {
let agent = ureq::builder()
.user_agent("clippy/lintcheck (github.com/rust-lang/rust-clippy/)")
.build();

let mut crates = Vec::with_capacity(number);
let mut query = "?sort=recent-downloads&per_page=100".to_string();
while crates.len() < number {
let page: Page = agent
.get(&format!("https://crates.io/api/v1/crates{query}"))
.call()?
.into_json()?;

query = page.meta.next_page;
crates.extend(page.crates);
crates.truncate(number);

let width = number.ilog10() as usize + 1;
println!("Fetched {:>width$}/{number} crates", crates.len());
}

let mut out = "[crates]\n".to_string();
for Crate { name, max_version } in crates {
writeln!(out, "{name} = {{ name = '{name}', versions = ['{max_version}'] }}").unwrap();
}
fs::write(output, out)?;

Ok(())
}

0 comments on commit 8065e0f

Please sign in to comment.