diff options
Diffstat (limited to 'src/tools/clippy/lintcheck')
| -rw-r--r-- | src/tools/clippy/lintcheck/Cargo.toml | 14 | ||||
| -rw-r--r-- | src/tools/clippy/lintcheck/README.md | 11 | ||||
| -rw-r--r-- | src/tools/clippy/lintcheck/src/config.rs | 160 | ||||
| -rw-r--r-- | src/tools/clippy/lintcheck/src/popular-crates.rs | 65 |
4 files changed, 142 insertions, 108 deletions
diff --git a/src/tools/clippy/lintcheck/Cargo.toml b/src/tools/clippy/lintcheck/Cargo.toml index 653121af5..27d32f390 100644 --- a/src/tools/clippy/lintcheck/Cargo.toml +++ b/src/tools/clippy/lintcheck/Cargo.toml @@ -8,12 +8,16 @@ repository = "https://github.com/rust-lang/rust-clippy" categories = ["development-tools"] edition = "2021" publish = false +default-run = "lintcheck" [dependencies] +anyhow = "1.0.69" cargo_metadata = "0.15.3" -clap = "4.1.4" +clap = { version = "4.1.8", features = ["derive", "env"] } +crates_io_api = "0.8.1" crossbeam-channel = "0.5.6" flate2 = "1.0" +indicatif = "0.17.3" rayon = "1.5.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.85" @@ -24,3 +28,11 @@ walkdir = "2.3" [features] deny-warnings = [] + +[[bin]] +name = "lintcheck" +path = "src/main.rs" + +[[bin]] +name = "popular-crates" +path = "src/popular-crates.rs" diff --git a/src/tools/clippy/lintcheck/README.md b/src/tools/clippy/lintcheck/README.md index 6142de5e3..faf3ce909 100644 --- a/src/tools/clippy/lintcheck/README.md +++ b/src/tools/clippy/lintcheck/README.md @@ -16,7 +16,7 @@ or cargo lintcheck ``` -By default the logs will be saved into +By default, the logs will be saved into `lintcheck-logs/lintcheck_crates_logs.txt`. You can set a custom sources.toml by adding `--crates-toml custom.toml` or using @@ -25,6 +25,15 @@ the repo root. The results will then be saved to `lintcheck-logs/custom_logs.toml`. +The `custom.toml` file may be built using <https://crates.io> recently most +downloaded crates by using the `popular-crates` binary from the `lintcheck` +directory. For example, to retrieve the 100 recently most downloaded crates: + +``` +cargo run --release --bin popular-crates -- -n 100 custom.toml +``` + + ### Configuring the Crate Sources The sources to check are saved in a `toml` file. There are three types of diff --git a/src/tools/clippy/lintcheck/src/config.rs b/src/tools/clippy/lintcheck/src/config.rs index e0244ddce..3f01e9bb0 100644 --- a/src/tools/clippy/lintcheck/src/config.rs +++ b/src/tools/clippy/lintcheck/src/config.rs @@ -1,131 +1,79 @@ -use clap::{Arg, ArgAction, ArgMatches, Command}; -use std::env; -use std::path::PathBuf; +use clap::Parser; +use std::{num::NonZeroUsize, path::PathBuf}; -fn get_clap_config() -> ArgMatches { - Command::new("lintcheck") - .about("run clippy on a set of crates and check output") - .args([ - Arg::new("only") - .action(ArgAction::Set) - .value_name("CRATE") - .long("only") - .help("Only process a single crate of the list"), - Arg::new("crates-toml") - .action(ArgAction::Set) - .value_name("CRATES-SOURCES-TOML-PATH") - .long("crates-toml") - .help("Set the path for a crates.toml where lintcheck should read the sources from"), - Arg::new("threads") - .action(ArgAction::Set) - .value_name("N") - .value_parser(clap::value_parser!(usize)) - .short('j') - .long("jobs") - .help("Number of threads to use, 0 automatic choice"), - Arg::new("fix") - .long("fix") - .help("Runs cargo clippy --fix and checks if all suggestions apply"), - Arg::new("filter") - .long("filter") - .action(ArgAction::Append) - .value_name("clippy_lint_name") - .help("Apply a filter to only collect specified lints, this also overrides `allow` attributes"), - Arg::new("markdown") - .long("markdown") - .help("Change the reports table to use markdown links"), - Arg::new("recursive") - .long("recursive") - .help("Run clippy on the dependencies of crates specified in crates-toml") - .conflicts_with("threads") - .conflicts_with("fix"), - ]) - .get_matches() -} - -#[derive(Debug, Clone)] +#[derive(Clone, Debug, Parser)] pub(crate) struct LintcheckConfig { - /// max number of jobs to spawn (default 1) + /// Number of threads to use (default: all unless --fix or --recursive) + #[clap( + long = "jobs", + short = 'j', + value_name = "N", + default_value_t = 0, + hide_default_value = true + )] pub max_jobs: usize, - /// we read the sources to check from here + /// Set the path for a crates.toml where lintcheck should read the sources from + #[clap( + long = "crates-toml", + value_name = "CRATES-SOURCES-TOML-PATH", + default_value = "lintcheck/lintcheck_crates.toml", + hide_default_value = true, + env = "LINTCHECK_TOML", + hide_env = true + )] pub sources_toml_path: PathBuf, - /// we save the clippy lint results here - pub lintcheck_results_path: PathBuf, - /// Check only a specified package + /// File to save the clippy lint results here + #[clap(skip = "")] + pub lintcheck_results_path: PathBuf, // Overridden in new() + /// Only process a single crate on the list + #[clap(long, value_name = "CRATE")] pub only: Option<String>, - /// whether to just run --fix and not collect all the warnings + /// Runs cargo clippy --fix and checks if all suggestions apply + #[clap(long, conflicts_with("max_jobs"))] pub fix: bool, - /// A list of lints that this lintcheck run should focus on + /// Apply a filter to only collect specified lints, this also overrides `allow` attributes + #[clap(long = "filter", value_name = "clippy_lint_name", use_value_delimiter = true)] pub lint_filter: Vec<String>, - /// Indicate if the output should support markdown syntax + /// Change the reports table to use markdown links + #[clap(long)] pub markdown: bool, - /// Run clippy on the dependencies of crates + /// Run clippy on the dependencies of crates specified in crates-toml + #[clap(long, conflicts_with("max_jobs"))] pub recursive: bool, } impl LintcheckConfig { pub fn new() -> Self { - let clap_config = get_clap_config(); - - // first, check if we got anything passed via the LINTCHECK_TOML env var, - // if not, ask clap if we got any value for --crates-toml <foo> - // if not, use the default "lintcheck/lintcheck_crates.toml" - let sources_toml = env::var("LINTCHECK_TOML").unwrap_or_else(|_| { - clap_config - .get_one::<String>("crates-toml") - .map_or("lintcheck/lintcheck_crates.toml", |s| &**s) - .into() - }); - - let markdown = clap_config.contains_id("markdown"); - let sources_toml_path = PathBuf::from(sources_toml); + let mut config = LintcheckConfig::parse(); // for the path where we save the lint results, get the filename without extension (so for // wasd.toml, use "wasd"...) - let filename: PathBuf = sources_toml_path.file_stem().unwrap().into(); - let lintcheck_results_path = PathBuf::from(format!( + let filename: PathBuf = config.sources_toml_path.file_stem().unwrap().into(); + config.lintcheck_results_path = PathBuf::from(format!( "lintcheck-logs/{}_logs.{}", filename.display(), - if markdown { "md" } else { "txt" } + if config.markdown { "md" } else { "txt" } )); - // look at the --threads arg, if 0 is passed, ask rayon rayon how many threads it would spawn and - // use half of that for the physical core count - // by default use a single thread - let max_jobs = match clap_config.get_one::<usize>("threads") { - Some(&0) => { - // automatic choice - // Rayon seems to return thread count so half that for core count - rayon::current_num_threads() / 2 - }, - Some(&threads) => threads, - // no -j passed, use a single thread - None => 1, + // look at the --threads arg, if 0 is passed, use the threads count + if config.max_jobs == 0 { + config.max_jobs = if config.fix || config.recursive { + 1 + } else { + std::thread::available_parallelism().map_or(1, NonZeroUsize::get) + }; }; - let lint_filter: Vec<String> = clap_config - .get_many::<String>("filter") - .map(|iter| { - iter.map(|lint_name| { - let mut filter = lint_name.replace('_', "-"); - if !filter.starts_with("clippy::") { - filter.insert_str(0, "clippy::"); - } - filter - }) - .collect() - }) - .unwrap_or_default(); - - LintcheckConfig { - max_jobs, - sources_toml_path, - lintcheck_results_path, - only: clap_config.get_one::<String>("only").map(String::from), - fix: clap_config.contains_id("fix"), - lint_filter, - markdown, - recursive: clap_config.contains_id("recursive"), + for lint_name in &mut config.lint_filter { + *lint_name = format!( + "clippy::{}", + lint_name + .strip_prefix("clippy::") + .unwrap_or(lint_name) + .replace('_', "-") + ); } + + config } } diff --git a/src/tools/clippy/lintcheck/src/popular-crates.rs b/src/tools/clippy/lintcheck/src/popular-crates.rs new file mode 100644 index 000000000..fdab984ad --- /dev/null +++ b/src/tools/clippy/lintcheck/src/popular-crates.rs @@ -0,0 +1,65 @@ +#![deny(clippy::pedantic)] + +use clap::Parser; +use crates_io_api::{CratesQueryBuilder, Sort, SyncClient}; +use indicatif::ProgressBar; +use std::collections::HashSet; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::PathBuf; +use std::time::Duration; + +#[derive(Parser)] +struct Opts { + /// Output TOML file name + output: PathBuf, + /// Number of crate names to download + #[clap(short, long, default_value_t = 100)] + number: usize, + /// Do not output progress + #[clap(short, long)] + quiet: bool, +} + +fn main() -> anyhow::Result<()> { + let opts = Opts::parse(); + let mut output = BufWriter::new(File::create(opts.output)?); + output.write_all(b"[crates]\n")?; + let client = SyncClient::new( + "clippy/lintcheck (github.com/rust-lang/rust-clippy/)", + Duration::from_secs(1), + )?; + let mut seen_crates = HashSet::new(); + let pb = if opts.quiet { + None + } else { + Some(ProgressBar::new(opts.number as u64)) + }; + let mut query = CratesQueryBuilder::new() + .sort(Sort::RecentDownloads) + .page_size(100) + .build(); + while seen_crates.len() < opts.number { + let retrieved = client.crates(query.clone())?.crates; + if retrieved.is_empty() { + eprintln!("No more than {} crates available from API", seen_crates.len()); + break; + } + for c in retrieved { + if seen_crates.insert(c.name.clone()) { + output.write_all( + format!( + "{} = {{ name = '{}', versions = ['{}'] }}\n", + c.name, c.name, c.max_version + ) + .as_bytes(), + )?; + if let Some(pb) = &pb { + pb.inc(1); + } + } + } + query.set_page(query.page() + 1); + } + Ok(()) +} |