From 04fed592b5dd9c9499c65cf36cdd984282cc80d1 Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 18 Jul 2022 11:44:20 +0100 Subject: [PATCH 1/5] Use miette for fancy error reporting --- Cargo.lock | 168 ++++++++++++++++++++++++++++++++++ tools/kbcheck/Cargo.toml | 1 + tools/kbcheck/src/data.rs | 35 ++----- tools/kbcheck/src/main.rs | 75 ++++++--------- tools/kbcheck/src/validate.rs | 15 ++- 5 files changed, 213 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d35e2c..3f71296 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -55,6 +70,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.0" @@ -100,6 +130,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + [[package]] name = "cfg-if" version = "1.0.0" @@ -264,6 +300,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + [[package]] name = "h2" version = "0.3.13" @@ -398,6 +440,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "is_ci" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" + [[package]] name = "iso8601" version = "0.4.2" @@ -457,6 +505,7 @@ version = "0.1.0" dependencies = [ "clap 3.2.12", "jsonschema", + "miette", "serde", "serde_json", "serde_yaml", @@ -514,6 +563,37 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miette" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d67f6972a70e33dbb5551875c6a3e46ae0a7cddd4661a2811ee48be51054e9" +dependencies = [ + "atty", + "backtrace", + "miette-derive", + "once_cell", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap 0.15.0", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "426594bc7266dedee4d687cdaebc121c74c52a667e4ce933c83694ad035990a6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.16" @@ -526,6 +606,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" @@ -650,6 +739,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.12.0" @@ -662,6 +760,12 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +[[package]] +name = "owo-colors" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" + [[package]] name = "parking_lot" version = "0.12.1" @@ -805,6 +909,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "ryu" version = "1.0.10" @@ -893,6 +1003,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "smawk" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" + [[package]] name = "socket2" version = "0.4.4" @@ -939,6 +1055,34 @@ dependencies = [ "syn", ] +[[package]] +name = "supports-color" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872ced36b91d47bae8a214a683fe54e7078875b399dfa251df346c9b547d1f9" +dependencies = [ + "atty", + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590b34f7c5f01ecc9d78dba4b3f445f31df750a67621cf31626f3b7441ce6406" +dependencies = [ + "atty", +] + +[[package]] +name = "supports-unicode" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8b945e45b417b125a8ec51f1b7df2f8df7920367700d1f98aedd21e5735f8b2" +dependencies = [ + "atty", +] + [[package]] name = "syn" version = "1.0.98" @@ -959,6 +1103,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -973,6 +1127,11 @@ name = "textwrap" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] [[package]] name = "thiserror" @@ -1101,6 +1260,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +[[package]] +name = "unicode-linebreak" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a52dcaab0c48d931f7cc8ef826fa51690a08e1ea55117ef26f89864f532383f" +dependencies = [ + "regex", +] + [[package]] name = "unicode-normalization" version = "0.1.19" diff --git a/tools/kbcheck/Cargo.toml b/tools/kbcheck/Cargo.toml index 7261639..aedbaad 100644 --- a/tools/kbcheck/Cargo.toml +++ b/tools/kbcheck/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] jsonschema="0.16" +miette={ version="5", features = ["fancy"] } serde="1" serde_json="1" serde_yaml="0.8" diff --git a/tools/kbcheck/src/data.rs b/tools/kbcheck/src/data.rs index 981887f..b6cdcd7 100644 --- a/tools/kbcheck/src/data.rs +++ b/tools/kbcheck/src/data.rs @@ -1,4 +1,5 @@ use crate::entry::Entry; +use miette::Diagnostic; use std::collections::BTreeMap; use std::fs::File; use std::io; @@ -9,32 +10,16 @@ use walkdir::{self, DirEntry, WalkDir}; pub type EntriesMap = BTreeMap; -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, Diagnostic)] pub enum DataError { - #[error("Could not open file")] - IoError( - #[from] - #[source] - io::Error, - ), - #[error("JSON parsing failed")] - JsonParseError( - #[from] - #[source] - serde_json::Error, - ), - #[error("Loading YAML failed")] - YamlLoadError( - #[from] - #[source] - serde_yaml::Error, - ), - #[error("JSON parsing failed")] - WalkDirError( - #[from] - #[source] - walkdir::Error, - ), + #[error(transparent)] + IoError(#[from] io::Error), + #[error(transparent)] + JsonParseError(#[from] serde_json::Error), + #[error(transparent)] + YamlLoadError(#[from] serde_yaml::Error), + #[error(transparent)] + WalkDirError(#[from] walkdir::Error), } /// Read a path into a serde_json::Value diff --git a/tools/kbcheck/src/main.rs b/tools/kbcheck/src/main.rs index 639fe7a..98250be 100644 --- a/tools/kbcheck/src/main.rs +++ b/tools/kbcheck/src/main.rs @@ -1,5 +1,6 @@ use clap::{Parser, Subcommand}; use kbcheck::{data, validate}; +use miette::{Diagnostic, IntoDiagnostic, Result}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -21,13 +22,7 @@ enum Commands { Validate, } -enum CommandStatus { - Ok, - Failure, - UnexpectedError, -} - -fn get_tags(root_path: &Path) -> Result, data::DataError> { +fn get_tags(root_path: &Path) -> Result> { let mut tags = BTreeMap::new(); for entry in data::load_all(root_path)?.values() { for tag in entry.tags.iter() { @@ -41,57 +36,43 @@ fn get_tags(root_path: &Path) -> Result, data::D Ok(tags) } -fn tags(root_path: &Path) -> CommandStatus { - match get_tags(root_path) { - Ok(tags) => { - let mut all_tags = tags.values().collect::>(); - all_tags.sort(); - for (name, count) in all_tags.iter() { - println!("{}:{}", name, count); - } - CommandStatus::Ok - } - Err(err) => { - println!("{}", err); - CommandStatus::UnexpectedError - } +fn tags(root_path: &Path) -> Result<()> { + let tags = get_tags(root_path)?; + let mut all_tags = tags.values().collect::>(); + all_tags.sort(); + for (name, count) in all_tags.iter() { + println!("{}:{}", name, count); } + Ok(()) } -fn validate(root_path: &Path) -> CommandStatus { - match validate::validate(root_path) { - Ok(errors) => { - if !errors.is_empty() { - println!("Validation failed"); - for err in errors { - println!("{}", err); - } - CommandStatus::Failure - } else { - CommandStatus::Ok - } - } - Err(err) => { +#[derive(thiserror::Error, Debug, Diagnostic)] +pub enum ValidateError { + #[error("Validation failed")] + ValidationFailed, + #[error(transparent)] + UnexpectedError(#[from] validate::ValidateError), +} + +fn validate(root_path: &Path) -> Result<()> { + let errors = validate::validate(root_path)?; + if !errors.is_empty() { + for err in errors { println!("{}", err); - CommandStatus::UnexpectedError } + Err(ValidateError::ValidationFailed.into()) + } else { + Ok(()) } } -fn run() -> CommandStatus { +fn main() -> Result<()> { let cli = Cli::parse(); let root_path = cli.root_path.unwrap_or_default(); match &cli.command { Commands::Tags => tags(&root_path), Commands::Validate => validate(&root_path), - } -} - -fn main() { - let exit_code = match run() { - CommandStatus::Ok => 0, - CommandStatus::Failure => 1, - CommandStatus::UnexpectedError => 2, - }; - std::process::exit(exit_code); + Commands::Bug { path } => bug(&path), + }?; + Ok(()) } diff --git a/tools/kbcheck/src/validate.rs b/tools/kbcheck/src/validate.rs index 6217416..2dbe144 100644 --- a/tools/kbcheck/src/validate.rs +++ b/tools/kbcheck/src/validate.rs @@ -1,6 +1,7 @@ use crate::data::{iter_data_files, read_json, EntriesMap}; use crate::entry::Entry; use jsonschema::JSONSchema; +use miette::Diagnostic; use serde::de::{Deserialize, IntoDeserializer}; use std::collections::BTreeMap; use std::fmt::Display; @@ -11,14 +12,10 @@ use url::Url; pub type Failures = Vec; -#[derive(Error, Debug)] -enum ValidateError { - #[error("Load failed")] - DataError( - #[from] - #[source] - crate::data::DataError, - ), +#[derive(Error, Debug, Diagnostic)] +pub enum ValidateError { + #[error(transparent)] + DataError(#[from] crate::data::DataError), #[error("Schema compile failed")] SchemaError, } @@ -183,7 +180,7 @@ fn global_validate(entries: EntriesMap) -> Failures { } /// Validate knowledge base entries -pub fn validate(root_path: &Path) -> Result> { +pub fn validate(root_path: &Path) -> Result { let mut errors: Failures = Vec::new(); let (entries, file_errors) = load_and_validate_files(root_path)?; errors.extend(file_errors.into_iter()); From 615359eacc105c5998289920b6eb26c480e5defc Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 19 Jul 2022 11:59:03 +0100 Subject: [PATCH 2/5] Add simple bugzilla client This provides structs for holding bug data from the Bugzilla REST API and a simple blocking client based on reqwest. --- Cargo.lock | 207 ++++++++++++++++++++++++++++++++++ tools/kbcheck/Cargo.toml | 1 + tools/kbcheck/src/bugzilla.rs | 172 ++++++++++++++++++++++++++++ tools/kbcheck/src/lib.rs | 1 + 4 files changed, 381 insertions(+) create mode 100644 tools/kbcheck/src/bugzilla.rs diff --git a/Cargo.lock b/Cargo.lock index 3f71296..892dc80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -196,6 +196,22 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -215,12 +231,36 @@ dependencies = [ "regex", ] +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -413,6 +453,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -434,6 +487,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.5.0" @@ -506,6 +568,7 @@ dependencies = [ "clap 3.2.12", "jsonschema", "miette", + "reqwest", "serde", "serde_json", "serde_yaml", @@ -627,6 +690,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.1" @@ -754,6 +835,51 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.1.0" @@ -807,6 +933,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -875,6 +1007,15 @@ version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqwest" version = "0.11.11" @@ -890,17 +1031,20 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-tls", "ipnet", "js-sys", "lazy_static", "log", "mime", + "native-tls", "percent-encoding", "pin-project-lite", "serde", "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tower-service", "url", "wasm-bindgen", @@ -930,12 +1074,45 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.139" @@ -1094,6 +1271,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -1202,6 +1393,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.3" @@ -1309,6 +1510,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/tools/kbcheck/Cargo.toml b/tools/kbcheck/Cargo.toml index aedbaad..d32328a 100644 --- a/tools/kbcheck/Cargo.toml +++ b/tools/kbcheck/Cargo.toml @@ -15,3 +15,4 @@ walkdir = "2" url={ version = "2", features = ["serde"] } thiserror = "1" clap = { version = "3", features = ["derive"] } +reqwest="0.11" \ No newline at end of file diff --git a/tools/kbcheck/src/bugzilla.rs b/tools/kbcheck/src/bugzilla.rs new file mode 100644 index 0000000..4c4880b --- /dev/null +++ b/tools/kbcheck/src/bugzilla.rs @@ -0,0 +1,172 @@ +use miette::Diagnostic; +use reqwest; +use serde::{Deserialize, Serialize}; +use serde_json::{self, Map, Value}; +use std::io::{self, Read}; +use thiserror; +use url::Url; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum OneOrMany { + One(T), + Many(Vec), +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Bug { + pub actual_time: Option, + // This is documented to return an array but b.m.o returns a string + pub alias: Option>, + pub assigned_to: String, + pub assigned_to_detail: User, + pub blocks: Vec, + pub cc: Vec, + pub cc_detail: Vec, + pub classification: String, + pub component: String, + pub creation_time: String, + pub creator: String, + pub creator_detail: User, + pub deadline: Option, + pub depends_on: Vec, + pub dupe_of: Option, + pub estimated_time: Option, + pub flags: Vec, + pub groups: Vec, + pub id: u64, + pub is_cc_accessible: bool, + pub is_confirmed: bool, + pub is_open: bool, + pub is_creator_accessible: bool, + pub keywords: Vec, + pub last_change_time: String, // datetime + pub op_sys: String, + pub platform: String, + pub priority: String, + pub qa_contact: String, + pub qa_contact_detail: Option, + pub remaining_time: Option, + pub resolution: String, + pub see_also: Vec, + pub severity: String, + pub status: String, + pub summary: String, + pub target_milestone: String, + pub update_token: Option, + pub url: String, + pub version: String, + pub whiteboard: String, + // Extra fields + pub tags: Option>, + pub duplicates: Option>, + // Instance-specific fields + #[serde(flatten)] + pub custom: Map, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct User { + pub id: u64, + pub real_name: String, + pub name: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Flag { + pub id: u64, + pub name: String, + pub type_id: u64, + pub creation_date: String, // datetime + pub modification_date: String, // datetime + pub status: String, + pub setter: String, + pub requestee: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Bugs { + pub bugs: Vec, + pub faults: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ErrorResponse { + pub code: u64, + pub documentation: String, + pub message: String, +} + +#[derive(thiserror::Error, Debug, Diagnostic)] +pub enum BugzillaError { + #[error(transparent)] + IoError(#[from] io::Error), + #[error(transparent)] + JsonParseError(#[from] serde_json::Error), + #[error(transparent)] + RequestError(#[from] reqwest::Error), + #[error("Bugzilla API Error")] + APIError { + code: u64, + documentation: String, + message: String, + }, + #[error("No bugs found")] + ZarroBoogs, +} + +impl From for BugzillaError { + fn from(err: ErrorResponse) -> Self { + BugzillaError::APIError { + code: err.code, + documentation: err.documentation, + message: err.message, + } + } +} + +pub struct Client { + base_url: Url, + http_client: reqwest::blocking::Client, +} + +impl Client { + pub fn new(base_url: Url) -> Client { + Client { + base_url, + http_client: reqwest::blocking::Client::new(), + } + } + + pub fn get_bug(&self, bug_id: u64) -> Result { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .expect("Can convert bugzilla URL") + .push("rest") + .push("bug") + .push(&bug_id.to_string()); + let mut resp = self.http_client.get(url).send()?; + + let mut resp_data = String::new(); + resp.read_to_string(&mut resp_data)?; + + if resp.status().is_success() { + let data: Bugs = serde_json::from_str(&resp_data)?; + if let Some(bug) = data.bugs.into_iter().next() { + Ok(bug) + } else { + Err(BugzillaError::ZarroBoogs) + } + } else { + let try_resp_err: Result = serde_json::from_str(&resp_data); + if let Ok(resp_err) = try_resp_err { + Err(resp_err.into()) + } else { + Err(resp + .error_for_status() + .expect_err("Response should be an error") + .into()) + } + } + } +} diff --git a/tools/kbcheck/src/lib.rs b/tools/kbcheck/src/lib.rs index a0b3842..891f5ca 100644 --- a/tools/kbcheck/src/lib.rs +++ b/tools/kbcheck/src/lib.rs @@ -1,3 +1,4 @@ +pub mod bugzilla; pub mod data; pub mod entry; pub mod validate; From d23106a951ebd5d3dc74935842064a6007821965 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 19 Jul 2022 12:01:21 +0100 Subject: [PATCH 3/5] Make Solutions fields pub --- tools/kbcheck/src/entry.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/kbcheck/src/entry.rs b/tools/kbcheck/src/entry.rs index c38670a..a30714d 100644 --- a/tools/kbcheck/src/entry.rs +++ b/tools/kbcheck/src/entry.rs @@ -23,11 +23,11 @@ pub enum UserBaseImpact { #[serde(deny_unknown_fields)] pub struct Solutions { #[serde(default)] - interventions: Vec, + pub interventions: Vec, #[serde(default)] - notes: Vec, + pub notes: Vec, #[serde(default)] - workarounds: Vec, + pub workarounds: Vec, } #[derive(Debug, Serialize, Deserialize, Default)] From dabecf2f61c50bef52363f06e12b2f0f3d816a07 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 19 Jul 2022 12:01:56 +0100 Subject: [PATCH 4/5] Add `updates` subcommand This implements checks for possible updates to the knowledge base from external data. At the moment there are two checks: has a platform_issues bug been closed as a dupe of another bug, and is there a `see_also` entry in one of the linked bugs that isn't in the `breakage` column. To be really useful there might need to be a way to allow list known issues. --- tools/kbcheck/src/lib.rs | 1 + tools/kbcheck/src/main.rs | 20 ++- tools/kbcheck/src/updates.rs | 245 +++++++++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 tools/kbcheck/src/updates.rs diff --git a/tools/kbcheck/src/lib.rs b/tools/kbcheck/src/lib.rs index 891f5ca..67130a8 100644 --- a/tools/kbcheck/src/lib.rs +++ b/tools/kbcheck/src/lib.rs @@ -1,4 +1,5 @@ pub mod bugzilla; pub mod data; pub mod entry; +pub mod updates; pub mod validate; diff --git a/tools/kbcheck/src/main.rs b/tools/kbcheck/src/main.rs index 98250be..b26d2bd 100644 --- a/tools/kbcheck/src/main.rs +++ b/tools/kbcheck/src/main.rs @@ -1,6 +1,6 @@ use clap::{Parser, Subcommand}; -use kbcheck::{data, validate}; -use miette::{Diagnostic, IntoDiagnostic, Result}; +use kbcheck::{data, updates, validate}; +use miette::{Diagnostic, Result}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -20,6 +20,7 @@ enum Commands { Tags, /// Validate data files against schema Validate, + Updates, } fn get_tags(root_path: &Path) -> Result> { @@ -66,13 +67,26 @@ fn validate(root_path: &Path) -> Result<()> { } } +fn updates(root_path: &Path) -> Result<()> { + let updates = updates::check_updates(root_path)?; + if !updates.is_empty() { + for (path, updates) in updates.iter() { + println!("{}", path.display()); + for update in updates.iter() { + println!(" {}\n Try: {}", update.error, update.suggestion); + } + } + } + Ok(()) +} + fn main() -> Result<()> { let cli = Cli::parse(); let root_path = cli.root_path.unwrap_or_default(); match &cli.command { Commands::Tags => tags(&root_path), Commands::Validate => validate(&root_path), - Commands::Bug { path } => bug(&path), + Commands::Updates => updates(&root_path), }?; Ok(()) } diff --git a/tools/kbcheck/src/updates.rs b/tools/kbcheck/src/updates.rs new file mode 100644 index 0000000..da835cc --- /dev/null +++ b/tools/kbcheck/src/updates.rs @@ -0,0 +1,245 @@ +use crate::bugzilla; +use crate::data::load_all; +use crate::entry::Entry; +use miette::Diagnostic; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use url::Url; + +#[derive(Error, Debug, Diagnostic)] +pub enum UpdateError { + #[error(transparent)] + DataError(#[from] crate::data::DataError), + #[error(transparent)] + BugzillaError(#[from] crate::bugzilla::BugzillaError), +} + +pub struct Update { + pub error: String, + pub suggestion: String, +} + +/// Extract a bugzilla bug id from a URL +fn bugzilla_id(url: &Url) -> Option { + if url.host_str() == Some("bugzilla.mozilla.org") && url.path() == "/show_bug.cgi" { + if let Some((_, id_str)) = url.query_pairs().find(|(name, _)| name == "id") { + return id_str.parse().ok(); + } + } + None +} + +/// Extract a webcompat issue number from a URL +fn webcompat_id(url: &Url) -> Option { + if (url.host_str() == Some("github.com") + && url.path().starts_with("/webcompat/web-bugs/issues/")) + || (url.host_str() == Some("webcompat.com") && url.path().starts_with("/issues/")) + { + if let Some(segments) = url.path_segments() { + if let Some(id) = segments.last() { + return id.parse().ok(); + } + } + } + None +} + +/// Check for possible updates relative to the bugzilla bugs listed in `references.platform_issues` +fn check_bug(bugzilla: &bugzilla::Client, entry: &Entry) -> Result, UpdateError> { + let mut updates = Vec::new(); + let platform_issue_ids = entry + .references + .platform_issues + .iter() + .filter_map(bugzilla_id); + let mut bugs_data = BTreeMap::new(); + for bug_id in platform_issue_ids { + let bug_data = bugzilla.get_bug(bug_id)?; + check_is_dupe(&bug_data, &mut updates); + bugs_data.insert(bug_id, bug_data); + } + check_missing_breakage(entry, &bugs_data, &mut updates); + Ok(updates) +} + +/// Check if the bug was closed as a dupe +fn check_is_dupe(bug_data: &bugzilla::Bug, updates: &mut Vec) { + if let Some(dupe) = bug_data.dupe_of { + updates.push(Update { + error: format!("Bug {} closed as a duplicate of {}", bug_data.id, dupe), + suggestion: format!( + "Update platform_issue entry to https://bugzilla.mozilla.org/show_bug.cgi?id={}", + dupe + ), + }); + } +} + +/// Check for see_also links to webcompat issues that aren't in the breakage data +fn check_missing_breakage( + entry: &Entry, + bugs_data: &BTreeMap, + updates: &mut Vec, +) { + let entry_breakage = + BTreeSet::from_iter(entry.references.breakage.iter().filter_map(webcompat_id)); + let mut bugs_see_also = BTreeSet::new(); + for bug_data in bugs_data.values() { + bugs_see_also.extend( + bug_data + .see_also + .iter() + .filter_map(|url_str| Url::parse(url_str).ok().as_ref().and_then(webcompat_id)), + ) + } + for webcompat_id in bugs_see_also.difference(&entry_breakage) { + updates.push(Update { + error: "Missing see also entry from linked bug in breakage".into(), + suggestion: format!( + "Add https://webcompat.com/issues/{} to references.breakage", + webcompat_id + ), + }); + } +} + +/// Check knowledge base entries against other data sources for possible updates +pub fn check_updates(root_path: &Path) -> Result>, UpdateError> { + let entries = load_all(root_path)?; + let bugzilla_client = bugzilla::Client::new( + Url::parse("https://bugzilla.mozilla.org").expect("Failed to parse bugzilla URL"), + ); + let mut updates = BTreeMap::new(); + for (path, entry) in entries.iter() { + let path_updates = check_bug(&bugzilla_client, entry)?; + if !path_updates.is_empty() { + updates.insert(path.to_owned(), path_updates); + } + } + Ok(updates) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::bugzilla::Bug; + use crate::entry::{Entry, References, Severity, Solutions}; + use std::default::Default; + + #[test] + fn test_bugzilla_id() { + assert_eq!( + bugzilla_id(&Url::parse("https://bugzilla.mozilla.org/show_bug.cgi?id=1234").unwrap()), + Some(1234) + ); + assert_eq!( + bugzilla_id( + &Url::parse("https://bugzilla.mozilla.org/show_bug.cgi?id=1234&foo=bar#baz") + .unwrap() + ), + Some(1234) + ); + assert_eq!( + bugzilla_id(&Url::parse("https://bugzilla.mozilla.org/other.cgi?id=1234").unwrap()), + None + ); + assert_eq!( + bugzilla_id(&Url::parse("https://bugzilla.mozilla.com/show_bug.cgi?id=1234").unwrap()), + None + ); + assert_eq!( + bugzilla_id(&Url::parse("https://bugzilla.mozilla.org/show_bug.cgi?id=1234a").unwrap()), + None + ); + } + + #[test] + fn test_webcompat_id() { + assert_eq!( + webcompat_id(&Url::parse("https://webcompat.com/issues/1234").unwrap()), + Some(1234) + ); + assert_eq!( + webcompat_id(&Url::parse("https://github.com/webcompat/web-bugs/issues/1234").unwrap()), + Some(1234) + ); + assert_eq!( + webcompat_id(&Url::parse("https://webcompat.com/issues/1234?foo=bar#baz").unwrap()), + Some(1234) + ); + assert_eq!( + webcompat_id( + &Url::parse("https://github.com/webcompat/web-bugs/issues/1234?foo=bar#baz") + .unwrap() + ), + Some(1234) + ); + assert_eq!( + webcompat_id(&Url::parse("https://other.webcompat.com/issues/1234").unwrap()), + None + ); + assert_eq!( + webcompat_id(&Url::parse("https://github.org/webcompat/web-bugs/issues/1234").unwrap()), + None + ); + assert_eq!( + webcompat_id(&Url::parse("https://webcompat.com/other/1234").unwrap()), + None + ); + assert_eq!( + webcompat_id(&Url::parse("https://github.com/webcompat/web-bugs/pulls/1234").unwrap()), + None + ); + assert_eq!( + webcompat_id(&Url::parse("https://github.com/other/web-bugs/issues/1234").unwrap()), + None + ); + } + + #[test] + fn test_is_dupe() { + let mut updates = Vec::new(); + let mut bug = Bug { + ..Default::default() + }; + check_is_dupe(&bug, &mut updates); + assert_eq!(updates.len(), 0); + bug.dupe_of = Some(1234); + check_is_dupe(&bug, &mut updates); + assert_eq!(updates.len(), 1); + } + + #[test] + fn test_missing_breakage() { + let mut updates = Vec::new(); + let bug = Bug { + see_also: vec![ + "https://webcompat.com/issues/1234".into(), + "https://webcompat.com/issues/1235".into(), + ], + ..Default::default() + }; + let entry = Entry { + title: "test".into(), + severity: Severity::Normal, + user_base_impact: None, + notes: None, + tags: vec![], + symptoms: vec![], + console_messages: vec![], + references: References { + breakage: vec![Url::parse("https://webcompat.com/issues/1234").unwrap()], + ..Default::default() + }, + solutions: Solutions { + ..Default::default() + }, + }; + let mut bugs_data = BTreeMap::new(); + bugs_data.insert(123, bug); + check_missing_breakage(&entry, &bugs_data, &mut updates); + assert_eq!(updates.len(), 1); + assert!(updates[0].suggestion.contains("1235")) + } +} From 855c9b1d5f0e77c1ad47cde911d47e759743de48 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 19 Jul 2022 12:20:42 +0100 Subject: [PATCH 5/5] Add gh action for running tooling tests --- .github/workflows/tools-tests.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/tools-tests.yml diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml new file mode 100644 index 0000000..4b5615b --- /dev/null +++ b/.github/workflows/tools-tests.yml @@ -0,0 +1,29 @@ +name: tools-tests + +on: + push: + branches: + - main + pull_request: +jobs: + rust-tests: + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@v3 + - name: "Cache" + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: "Rust toolchain" + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: "Test" + run: cargo test