From f5a7f63f942264ea3bda8ccacaad930d4141ad4b Mon Sep 17 00:00:00 2001 From: Niklas Wojtkowiak Date: Fri, 7 Jun 2024 15:30:49 +0300 Subject: [PATCH 1/2] Add Basic Syncing --- dns/Cargo.lock | 331 ++++++++++++++++++++++++++++++ dns/Cargo.toml | 3 + dns/domains/conf | 4 + dns/domains/db | Bin 0 -> 524287 bytes dns/domains/snap.0000000000000060 | Bin 0 -> 69 bytes dns/src/config/mod.rs | 43 +++- dns/src/config/structs.rs | 6 + dns/src/main.rs | 23 ++- dns/src/sync.rs | 66 ++++++ 9 files changed, 467 insertions(+), 9 deletions(-) create mode 100644 dns/domains/conf create mode 100644 dns/domains/db create mode 100644 dns/domains/snap.0000000000000060 create mode 100644 dns/src/sync.rs diff --git a/dns/Cargo.lock b/dns/Cargo.lock index c95ff77f..eafe47f5 100644 --- a/dns/Cargo.lock +++ b/dns/Cargo.lock @@ -582,6 +582,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -817,6 +827,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "flate2" version = "1.0.30" @@ -833,6 +859,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1089,6 +1130,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.8.0" @@ -1107,6 +1159,43 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.7", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1269,6 +1358,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "local-channel" version = "0.1.5" @@ -1422,6 +1517,23 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "no-std-compat" version = "0.4.1" @@ -1474,6 +1586,50 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1751,6 +1907,46 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -1810,6 +2006,19 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.21.12" @@ -1853,6 +2062,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1869,6 +2087,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -2133,6 +2374,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -2145,6 +2413,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "term" version = "0.7.0" @@ -2261,6 +2541,16 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -2319,6 +2609,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.40" @@ -2384,6 +2680,12 @@ dependencies = [ "trust-dns-proto", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typed-builder" version = "0.10.0" @@ -2467,12 +2769,27 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2504,6 +2821,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -2567,7 +2896,9 @@ dependencies = [ "prettytable", "rand", "regex", + "reqwest", "serde", + "serde_json", "sled", "tokio", "toml", diff --git a/dns/Cargo.toml b/dns/Cargo.toml index 7b296d9d..8bbd96a0 100644 --- a/dns/Cargo.toml +++ b/dns/Cargo.toml @@ -18,8 +18,11 @@ prettytable = "0.10.0" actix-governor = "0.5.0" pretty_env_logger = "0.5.0" clap-verbosity-flag = "2.2.0" +serde_json = "1.0.96" tokio = { version = "1.38.0", features = ["full"] } clap = { version = "4.5.4", features = ["derive"] } rand = { version = "0.8.5", features = ["small_rng"] } serde = { version = "1.0.203", features = ["derive"] } +reqwest = { version = "0.11", features = ["json"]} + diff --git a/dns/domains/conf b/dns/domains/conf new file mode 100644 index 00000000..4154d7c4 --- /dev/null +++ b/dns/domains/conf @@ -0,0 +1,4 @@ +segment_size: 524288 +use_compression: false +version: 0.34 +vQÁ \ No newline at end of file diff --git a/dns/domains/db b/dns/domains/db new file mode 100644 index 0000000000000000000000000000000000000000..c4d313c7775b17c65ef6795ff695c1888ed4573d GIT binary patch literal 524287 zcmeIuu?fNe5CzbSVhA?k2)c-)1R=2sh(y6o(b^$&j0~WKjeA>c{8nLU_NqJNVbU zS92M>+H9iDcCxuW)6c#*o!7l2lDLod$1ID^w-AqUJ Self { let default_offensive_words = vec!["nigg", "sex", "porn", "igg"]; let default_tld_list = vec![ - "mf", "btw", "fr", "yap", "dev", "scam", "zip", "root", "web", "rizz", "habibi", "sigma", "now", "it", "soy", "lol", "uwu", "ohio", "cat", + "mf", "btw", "fr", "yap", "dev", "scam", "zip", "root", "web", "rizz", "habibi", + "sigma", "now", "it", "soy", "lol", "uwu", "ohio", "cat", ]; Config { @@ -31,15 +32,37 @@ impl Config { }, settings: Settings { tld_list: default_tld_list.iter().map(|s| s.to_string()).collect(), - offensive_words: default_offensive_words.iter().map(|s| s.to_string()).collect(), + offensive_words: default_offensive_words + .iter() + .map(|s| s.to_string()) + .collect(), + }, + dns: Dns { + sync_from_list: vec![], }, } } - pub fn read(&self) -> Self { file::read(&self.config_path) } - pub fn get_address(&self) -> String { format!("{}:{}", self.server.address.clone(), self.server.port) } - pub fn tld_list(&self) -> Vec<&str> { self.settings.tld_list.iter().map(AsRef::as_ref).collect::>() } - pub fn offen_words(&self) -> Vec<&str> { self.settings.offensive_words.iter().map(AsRef::as_ref).collect::>() } + pub fn read(&self) -> Self { + file::read(&self.config_path) + } + pub fn get_address(&self) -> String { + format!("{}:{}", self.server.address.clone(), self.server.port) + } + pub fn tld_list(&self) -> Vec<&str> { + self.settings + .tld_list + .iter() + .map(AsRef::as_ref) + .collect::>() + } + pub fn offen_words(&self) -> Vec<&str> { + self.settings + .offensive_words + .iter() + .map(AsRef::as_ref) + .collect::>() + } pub fn set_path(&mut self, config_path: &String) -> &mut Self { self.config_path = config_path.clone(); @@ -53,7 +76,11 @@ impl Config { }; if let Err(err) = write(&self.config_path, contents) { - crashln!("Error writing config to {}.\n{}", self.config_path, string!(err).white()) + crashln!( + "Error writing config to {}.\n{}", + self.config_path, + string!(err).white() + ) } log::info!("Created config: {}", &self.config_path,); diff --git a/dns/src/config/structs.rs b/dns/src/config/structs.rs index 2360c6ce..25746e0d 100644 --- a/dns/src/config/structs.rs +++ b/dns/src/config/structs.rs @@ -6,6 +6,7 @@ pub struct Config { pub config_path: String, pub(crate) server: Server, pub(crate) settings: Settings, + pub(crate) dns: Dns, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -28,3 +29,8 @@ pub struct Settings { pub(crate) tld_list: Vec, pub(crate) offensive_words: Vec, } + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Dns { + pub(crate) sync_from_list: Vec, +} diff --git a/dns/src/main.rs b/dns/src/main.rs index 1cf94be6..012ba8b1 100644 --- a/dns/src/main.rs +++ b/dns/src/main.rs @@ -3,16 +3,21 @@ mod config; mod http; mod kv; mod secret; +mod sync; use clap::{Parser, Subcommand}; use clap_verbosity_flag::{LogLevel, Verbosity}; use config::Config; +use futures::executor::block_on; use macros_rs::fs::file_exists; +use tokio::runtime::Runtime; #[derive(Copy, Clone, Debug, Default)] struct Info; impl LogLevel for Info { - fn default() -> Option { Some(log::Level::Info) } + fn default() -> Option { + Some(log::Level::Info) + } } #[derive(Parser)] @@ -34,6 +39,8 @@ enum Commands { #[command(subcommand)] command: Key, }, + /// Sync all domains + Sync, } #[derive(Subcommand)] @@ -67,7 +74,16 @@ enum Key { }, } +#[derive(Subcommand)] +enum Sync { + /// Sync all keys + // #[command(visible_alias = "s")] + Sync, +} + fn main() { + let rt = Runtime::new().expect("failed to create Tokio runtime"); + rt.block_on(sync::sync()); let cli = Cli::parse(); let mut env = pretty_env_logger::formatted_builder(); let level = cli.verbose.log_level_filter(); @@ -93,5 +109,10 @@ fn main() { Key::Delete { name } => cli::remove(&cli, name), Key::Export { filename } => cli::export(&cli, filename), }, + Commands::Sync => { + if let Err(err) = rt.block_on(sync::sync()) { + log::error!("Failed to sync data: {err}"); + } + } }; } diff --git a/dns/src/sync.rs b/dns/src/sync.rs new file mode 100644 index 00000000..774eaa08 --- /dev/null +++ b/dns/src/sync.rs @@ -0,0 +1,66 @@ +// fetch GET /domains from the config.dns.sync_from_list +// and add them to the database if they don't exist + +use crate::config::Config; +use crate::http::Domain; +use macros_rs::fmt::{crashln, string}; +use mongodb::{bson::doc, options::FindOptions}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +struct Domains { + domains: Vec, + page: u32, + limit: u32, +} + +pub async fn sync() -> std::io::Result<()> { + let config_path = ("config.toml".to_string()); + + let config = Config::new().set_path(&config_path).read(); + let mut sync_list = config.dns.sync_from_list.clone(); + sync_list.sort(); + + let db = match config.connect_to_mongo().await { + Ok(client) => client, + Err(err) => crashln!("Failed to connect to MongoDB.\n{}", string!(err)), + }; + + // fetch GET /domains for each sync_list item + // and add them to the database if they don't exist + println!("syncing domains from {:?}", sync_list); + // {"domains":[{"tld":"btw","ip":"https://github.com/Smartlinuxcoder/iusearch.btw","name":"iusearch"},{"tld":"rizz","ip":"https://github.com/illy-dev/website-for-bussin-web","name":"skibidi"}],"page":1,"limit":2} + for item in sync_list { + let mut page = 1; + let mut domains: Vec = vec![]; + + loop { + let url = format!("https://{}/domains?limit=100&page={}", item, page); + println!("fetching {}", url); + let res = reqwest::get(&url).await.unwrap(); + println!("fetched"); + let body = res.text().await.unwrap(); + println!("{}", body); + //parts domains, page, limit + let domains = serde_json::from_str::(&body).unwrap(); + let len = &domains.domains.len(); + for domain in domains.domains { + if !db + .find_one(doc! { "name": &domain.name, "tld": &domain.tld }, None) + .await + .unwrap() + .is_some() + { + db.insert_one(&domain, None).await.unwrap(); + } + } + page += 1; + if page > 100 { + break; + } else if *len == 0 { + break; + } + } + } + Ok(()) +} From 8d8856627f97d9319f7c2360721f20dc56465259 Mon Sep 17 00:00:00 2001 From: Niklas Wojtkowiak Date: Fri, 7 Jun 2024 22:06:14 +0300 Subject: [PATCH 2/2] fix things --- dns/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dns/src/main.rs b/dns/src/main.rs index 012ba8b1..d3fc2eaf 100644 --- a/dns/src/main.rs +++ b/dns/src/main.rs @@ -83,7 +83,7 @@ enum Sync { fn main() { let rt = Runtime::new().expect("failed to create Tokio runtime"); - rt.block_on(sync::sync()); + // rt.block_on(sync::sync()); let cli = Cli::parse(); let mut env = pretty_env_logger::formatted_builder(); let level = cli.verbose.log_level_filter();