awesome/awesome-rust
1
0
Fork 0
mirror of https://github.com/rust-unofficial/awesome-rust.git synced 2025-03-03 14:32:38 +02:00
Code Issues Releases Activity

Improve github regex match

Browse Source
This commit is contained in:
Tom Parker-Shemilt 2021-10-16 22:20:07 +01:00
parent f13e5fe855
commit fa9b5f30e8
1 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/main.rs
View File

@ -161,7 +161,7 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>)
lazy_static! { lazy_static! {
static ref GITHUB_REPO_REGEX: Regex = static ref GITHUB_REPO_REGEX: Regex =
Regex::new(r"^https://github.com/(?P<org>[^/]+)/(?P<repo>[^/]+)/?$").unwrap(); Regex::new(r"^https://github.com/(?P<org>[^/]+)/(?P<repo>[^/]+)(.*)").unwrap();
static ref GITHUB_API_REGEX: Regex = Regex::new(r"https://api.github.com/").unwrap(); static ref GITHUB_API_REGEX: Regex = Regex::new(r"https://api.github.com/").unwrap();
static ref CRATE_REGEX: Regex = static ref CRATE_REGEX: Regex =
Regex::new(r"https://crates.io/crates/(?P<crate>[^/]+)/?$").unwrap(); Regex::new(r"https://crates.io/crates/(?P<crate>[^/]+)/?$").unwrap();
@ -392,13 +392,6 @@ async fn main() -> Result<(), Error> {
cargo_downloads: BTreeMap::new(), cargo_downloads: BTreeMap::new(),
}); });
// Overrides for popularity count, reasons at the top of the file
for url in POPULARITY_OVERRIDES.iter() {
popularity_data
.github_stars
.insert(url.clone(), MINIMUM_GITHUB_STARS);
}
let mut url_checks = vec![]; let mut url_checks = vec![];
let min_between_checks: Duration = Duration::days(3); let min_between_checks: Duration = Duration::days(3);
@ -450,25 +443,35 @@ async fn main() -> Result<(), Error> {
Tag::Link(_link_type, url, _title) | Tag::Image(_link_type, url, _title) => { Tag::Link(_link_type, url, _title) | Tag::Image(_link_type, url, _title) => {
if !url.starts_with("#") { if !url.starts_with("#") {
let new_url = url.to_string(); let new_url = url.to_string();
let existing = popularity_data.github_stars.get(&new_url); if POPULARITY_OVERRIDES.contains(&new_url) {
if let Some(stars) = existing { github_stars = Some(MINIMUM_GITHUB_STARS);
// Use existing star data, but re-retrieve url to check aliveness } else if GITHUB_REPO_REGEX.is_match(&url) {
// Some will have overrides, so don't check the regex yet let github_url = GITHUB_REPO_REGEX
github_stars = Some(*stars) .replace_all(&url, "https://github.com/$org/$repo")
} else if GITHUB_REPO_REGEX.is_match(&url) && existing.is_none() { .to_string();
github_stars = get_stars(&url).await; let existing = popularity_data.github_stars.get(&github_url);
if let Some(raw_stars) = github_stars { if let Some(stars) = existing {
popularity_data.github_stars.insert(new_url, raw_stars); // Use existing star data, but re-retrieve url to check aliveness
if raw_stars >= required_stars { // Some will have overrides, so don't check the regex yet
fs::write( github_stars = Some(*stars)
"results/popularity.yaml", } else {
serde_yaml::to_string(&popularity_data)?, github_stars = get_stars(&github_url).await;
)?; if let Some(raw_stars) = github_stars {
popularity_data
.github_stars
.insert(github_url.to_string(), raw_stars);
if raw_stars >= required_stars {
fs::write(
"results/popularity.yaml",
serde_yaml::to_string(&popularity_data)?,
)?;
}
link_count += 1;
continue;
} }
} }
link_count += 1; }
continue; if CRATE_REGEX.is_match(&url) {
} else if CRATE_REGEX.is_match(&url) {
let existing = popularity_data.cargo_downloads.get(&new_url); let existing = popularity_data.cargo_downloads.get(&new_url);
if let Some(downloads) = existing { if let Some(downloads) = existing {
cargo_downloads = Some(*downloads); cargo_downloads = Some(*downloads);