Add Basic Syncing

dns/Cargo.toml

@@ -18,8 +18,11 @@ prettytable = "0.10.0"
 actix-governor = "0.5.0"
 pretty_env_logger = "0.5.0"
 clap-verbosity-flag = "2.2.0"
+serde_json = "1.0.96"
 
 tokio = { version = "1.38.0", features = ["full"] }
 clap = { version = "4.5.4", features = ["derive"] }
 rand = { version = "0.8.5", features = ["small_rng"] }
 serde = { version = "1.0.203", features = ["derive"] }
+reqwest = { version = "0.11", features = ["json"]}
+

dns/domains/conf

@@ -0,0 +1,4 @@
+segment_size: 524288
+use_compression: false
+version: 0.34
+vQ�

dns/src/config/mod.rs

@@ -6,15 +6,16 @@ use colored::Colorize;
 use macros_rs::fmt::{crashln, string};
 use mongodb::{error::Error, options::ClientOptions, Client, Collection};
 use std::fs::write;
-use structs::{Mongo, Server, Settings};
+use structs::{Dns, Mongo, Server, Settings};
 
 pub use structs::Config;
 
 impl Config {
     pub fn new() -> Self {
         let default_offensive_words = vec!["nigg", "sex", "porn", "igg"];
         let default_tld_list = vec![
-            "mf", "btw", "fr", "yap", "dev", "scam", "zip", "root", "web", "rizz", "habibi", "sigma", "now", "it", "soy", "lol", "uwu", "ohio", "cat",
+            "mf", "btw", "fr", "yap", "dev", "scam", "zip", "root", "web", "rizz", "habibi",
+            "sigma", "now", "it", "soy", "lol", "uwu", "ohio", "cat",
         ];
 
         Config {
@@ -31,15 +32,37 @@ impl Config {
             },
             settings: Settings {
                 tld_list: default_tld_list.iter().map(|s| s.to_string()).collect(),
-                offensive_words: default_offensive_words.iter().map(|s| s.to_string()).collect(),
+                offensive_words: default_offensive_words
+                    .iter()
+                    .map(|s| s.to_string())
+                    .collect(),
+            },
+            dns: Dns {
+                sync_from_list: vec![],
             },
         }
     }
 
-    pub fn read(&self) -> Self { file::read(&self.config_path) }
-    pub fn get_address(&self) -> String { format!("{}:{}", self.server.address.clone(), self.server.port) }
-    pub fn tld_list(&self) -> Vec<&str> { self.settings.tld_list.iter().map(AsRef::as_ref).collect::<Vec<&str>>() }
-    pub fn offen_words(&self) -> Vec<&str> { self.settings.offensive_words.iter().map(AsRef::as_ref).collect::<Vec<&str>>() }
+    pub fn read(&self) -> Self {
+        file::read(&self.config_path)
+    }
+    pub fn get_address(&self) -> String {
+        format!("{}:{}", self.server.address.clone(), self.server.port)
+    }
+    pub fn tld_list(&self) -> Vec<&str> {
+        self.settings
+            .tld_list
+            .iter()
+            .map(AsRef::as_ref)
+            .collect::<Vec<&str>>()
+    }
+    pub fn offen_words(&self) -> Vec<&str> {
+        self.settings
+            .offensive_words
+            .iter()
+            .map(AsRef::as_ref)
+            .collect::<Vec<&str>>()
+    }
 
     pub fn set_path(&mut self, config_path: &String) -> &mut Self {
         self.config_path = config_path.clone();
@@ -53,7 +76,11 @@ impl Config {
         };
 
         if let Err(err) = write(&self.config_path, contents) {
-            crashln!("Error writing config to {}.\n{}", self.config_path, string!(err).white())
+            crashln!(
+                "Error writing config to {}.\n{}",
+                self.config_path,
+                string!(err).white()
+            )
         }
 
         log::info!("Created config: {}", &self.config_path,);

dns/src/config/structs.rs

@@ -6,6 +6,7 @@ pub struct Config {
     pub config_path: String,
     pub(crate) server: Server,
     pub(crate) settings: Settings,
+    pub(crate) dns: Dns,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -28,3 +29,8 @@ pub struct Settings {
     pub(crate) tld_list: Vec<String>,
     pub(crate) offensive_words: Vec<String>,
 }
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Dns {
+    pub(crate) sync_from_list: Vec<String>,
+}

dns/src/main.rs

@@ -3,16 +3,21 @@ mod config;
 mod http;
 mod kv;
 mod secret;
+mod sync;
 
 use clap::{Parser, Subcommand};
 use clap_verbosity_flag::{LogLevel, Verbosity};
 use config::Config;
+use futures::executor::block_on;
 use macros_rs::fs::file_exists;
+use tokio::runtime::Runtime;
 
 #[derive(Copy, Clone, Debug, Default)]
 struct Info;
 impl LogLevel for Info {
-    fn default() -> Option<log::Level> { Some(log::Level::Info) }
+    fn default() -> Option<log::Level> {
+        Some(log::Level::Info)
+    }
 }
 
 #[derive(Parser)]
@@ -34,6 +39,8 @@ enum Commands {
         #[command(subcommand)]
         command: Key,
     },
+    /// Sync all domains
+    Sync,
 }
 
 #[derive(Subcommand)]
@@ -67,7 +74,16 @@ enum Key {
     },
 }
 
+#[derive(Subcommand)]
+enum Sync {
+    /// Sync all keys
+    // #[command(visible_alias = "s")]
+    Sync,
+}
+
 fn main() {
+    let rt = Runtime::new().expect("failed to create Tokio runtime");
+    // rt.block_on(sync::sync());
     let cli = Cli::parse();
     let mut env = pretty_env_logger::formatted_builder();
     let level = cli.verbose.log_level_filter();
@@ -93,5 +109,10 @@ fn main() {
             Key::Delete { name } => cli::remove(&cli, name),
             Key::Export { filename } => cli::export(&cli, filename),
         },
+        Commands::Sync => {
+            if let Err(err) = rt.block_on(sync::sync()) {
+                log::error!("Failed to sync data: {err}");
+            }
+        }
     };
 }

dns/src/sync.rs

@@ -0,0 +1,66 @@
+// fetch GET /domains from the config.dns.sync_from_list
+// and add them to the database if they don't exist
+
+use crate::config::Config;
+use crate::http::Domain;
+use macros_rs::fmt::{crashln, string};
+use mongodb::{bson::doc, options::FindOptions};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Domains {
+    domains: Vec<Domain>,
+    page: u32,
+    limit: u32,
+}
+
+pub async fn sync() -> std::io::Result<()> {
+    let config_path = ("config.toml".to_string());
+
+    let config = Config::new().set_path(&config_path).read();
+    let mut sync_list = config.dns.sync_from_list.clone();
+    sync_list.sort();
+
+    let db = match config.connect_to_mongo().await {
+        Ok(client) => client,
+        Err(err) => crashln!("Failed to connect to MongoDB.\n{}", string!(err)),
+    };
+
+    // fetch GET /domains for each sync_list item
+    // and add them to the database if they don't exist
+    println!("syncing domains from {:?}", sync_list);
+    // {"domains":[{"tld":"btw","ip":"https://github.com/Smartlinuxcoder/iusearch.btw","name":"iusearch"},{"tld":"rizz","ip":"https://github.com/illy-dev/website-for-bussin-web","name":"skibidi"}],"page":1,"limit":2}
+    for item in sync_list {
+        let mut page = 1;
+        let mut domains: Vec<Domain> = vec![];
+
+        loop {
+            let url = format!("https://{}/domains?limit=100&page={}", item, page);
+            println!("fetching {}", url);
+            let res = reqwest::get(&url).await.unwrap();
+            println!("fetched");
+            let body = res.text().await.unwrap();
+            println!("{}", body);
+            //parts domains, page, limit
+            let domains = serde_json::from_str::<Domains>(&body).unwrap();
+            let len = &domains.domains.len();
+            for domain in domains.domains {
+                if !db
+                    .find_one(doc! { "name": &domain.name, "tld": &domain.tld }, None)
+                    .await
+                    .unwrap()
+                    .is_some()
+                {
+                    db.insert_one(&domain, None).await.unwrap();
+                }
+            }
+            page += 1;
+            if page > 100 {
+                break;
+            } else if *len == 0 {
+                break;
+            }
+        }
+    }
+    Ok(())
+}