From 0b3f03b3fd1cf4852a8205a0e874741d9d46e781 Mon Sep 17 00:00:00 2001 From: Katie Garwood Date: Mon, 26 May 2025 16:15:11 -0700 Subject: [PATCH 1/5] get cameratraps path from main function previously adding this to the path was in detect_img for some reason --- cougarvision_utils/detect_img.py | 5 ----- fetch_and_alert.py | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cougarvision_utils/detect_img.py b/cougarvision_utils/detect_img.py index c64a384..7612fd1 100644 --- a/cougarvision_utils/detect_img.py +++ b/cougarvision_utils/detect_img.py @@ -27,11 +27,6 @@ from cougarvision_visualize.visualize_helper import create_folder -with open("config/cameratraps.yml", 'r') as stream: - camera_traps_config = yaml.safe_load(stream) - sys.path.append(camera_traps_config['camera_traps_path']) - - def detect(images, config, c_model, classes, d_model): ''' This function takes in a dataframe of images and runs a detector model, diff --git a/fetch_and_alert.py b/fetch_and_alert.py index fa5e608..68cece3 100644 --- a/fetch_and_alert.py +++ b/fetch_and_alert.py @@ -46,6 +46,9 @@ # Load Configuration Settings from YML file with open(CONFIG_FILE, 'r', encoding='utf-8') as stream: CONFIG = yaml.safe_load(stream) +with open("config/cameratraps.yml", 'r') as stream: + camera_traps_config = yaml.safe_load(stream) + sys.path.append(camera_traps_config['camera_traps_path']) # Set Email Variables for fetching USERNAME = CONFIG['username'] PASSWORD = CONFIG['password'] From dd8a68f59a5be15492d3e394b7ae7a6d1cfd4bfc Mon Sep 17 00:00:00 2001 From: Katie Garwood Date: Mon, 26 May 2025 16:29:26 -0700 Subject: [PATCH 2/5] removed unecessary function to check and create new folder --- cougarvision_utils/detect_img.py | 8 ++++---- cougarvision_utils/get_images.py | 8 ++++---- cougarvision_visualize/visualize_helper.py | 12 ------------ 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/cougarvision_utils/detect_img.py b/cougarvision_utils/detect_img.py index 7612fd1..a371385 100644 --- a/cougarvision_utils/detect_img.py +++ b/cougarvision_utils/detect_img.py @@ -20,11 +20,11 @@ from animl import inference, split from sageranger import is_target, attach_image, post_event from animl.detect import detect_MD_batch, parse_MD +import os from cougarvision_utils.cropping import draw_bounding_box_on_image from cougarvision_utils.alert import smtp_setup, send_alert from cougarvision_visualize.visualize_helper import get_last_file_number -from cougarvision_visualize.visualize_helper import create_folder def detect(images, config, c_model, classes, d_model): @@ -122,10 +122,10 @@ def detect(images, config, c_model, classes, d_model): img.save(image_bytes, format="JPEG") img_byte = image_bytes.getvalue() if visualize_output is True: - folder_path = create_folder(labeled_img) - last_file_number = get_last_file_number(folder_path) + os.makedirs(labeled_img, exist_ok=True) + last_file_number = get_last_file_number(labeled_img) new_file_number = last_file_number + 1 - new_file_name = f"{folder_path}/image_{new_file_number}.jpg" + new_file_name = f"{labeled_img}/image_{new_file_number}.jpg" with open(new_file_name, "wb") as folder: folder.write(img_byte) diff --git a/cougarvision_utils/get_images.py b/cougarvision_utils/get_images.py index 5fe58d3..9ecee47 100644 --- a/cougarvision_utils/get_images.py +++ b/cougarvision_utils/get_images.py @@ -15,11 +15,11 @@ import json import urllib.request import os.path +import os import logging import requests import numpy as np from cougarvision_visualize.visualize_helper import get_last_file_number -from cougarvision_visualize.visualize_helper import create_folder ''' @@ -153,9 +153,9 @@ def fetch_image_api(config): info['file_thumb_url'], newname]) if visualize_output is True: - file_path = create_folder(unlabeled_img) - newname = file_path + 'image' - new_file_num = get_last_file_number(file_path) + os.makedirs(unlabeled_img, exist_ok=True) + newname = unlabeled_img + 'image' + new_file_num = get_last_file_number(unlabeled_img) new_file_num = new_file_num + 1 new_file_num = str(new_file_num) newname += "_" + new_file_num diff --git a/cougarvision_visualize/visualize_helper.py b/cougarvision_visualize/visualize_helper.py index 5b8c8d8..b44fafe 100644 --- a/cougarvision_visualize/visualize_helper.py +++ b/cougarvision_visualize/visualize_helper.py @@ -10,16 +10,4 @@ def get_last_file_number(folder_path): if num: # If there are digits in the filename max_num = max(max_num, int(num[-1])) # Use the last set of digits as the number return max_num - - -def create_folder(folder_path): - """ - Check if a folder exists at the specified path, and create it if it doesn't. - - folder_path: The path of the folder to check and potentially create. - return: path - """ - if not os.path.exists(folder_path): - os.makedirs(folder_path) - return folder_path From 7b6e52052c1be40a90c9f743f294afcc4d3e1165 Mon Sep 17 00:00:00 2001 From: Katie Garwood Date: Mon, 26 May 2025 16:35:17 -0700 Subject: [PATCH 3/5] move everything to main rather than being free in the script all the config and logging and loading was just happening when the script ran but it should just happen in main --- fetch_and_alert.py | 85 ++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/fetch_and_alert.py b/fetch_and_alert.py index 68cece3..5e4e6cf 100644 --- a/fetch_and_alert.py +++ b/fetch_and_alert.py @@ -35,34 +35,6 @@ from animl import megadetector -# Numpy FutureWarnings from tensorflow import -warnings.filterwarnings('ignore', category=FutureWarning) -# Parse arguments -PARSER = argparse.ArgumentParser(description='Retrieves images from \ - email & web scraper & runs detection') -PARSER.add_argument('config', type=str, help='Path to config file') -ARGS = PARSER.parse_args() -CONFIG_FILE = ARGS.config -# Load Configuration Settings from YML file -with open(CONFIG_FILE, 'r', encoding='utf-8') as stream: - CONFIG = yaml.safe_load(stream) -with open("config/cameratraps.yml", 'r') as stream: - camera_traps_config = yaml.safe_load(stream) - sys.path.append(camera_traps_config['camera_traps_path']) -# Set Email Variables for fetching -USERNAME = CONFIG['username'] -PASSWORD = CONFIG['password'] -TOKEN = CONFIG['token'] -AUTH = CONFIG['authorization'] -CLASSIFIER = CONFIG['classifier_model'] -DETECTOR = CONFIG['detector_model'] -DEV_EMAILS = CONFIG['dev_emails'] -HOST = 'imap.gmail.com' -RUN_SCHEDULER = CONFIG['run_scheduler'] -VISUALIZE_OUTPUT = CONFIG['visualize_output'] -LABELS = CONFIG['classes'] - - def logger(): '''Function to define logging file parameters''' msg_intro = "%(levelname)s:%(asctime)s:%(module)s:%(funcName)s:" @@ -73,22 +45,6 @@ def logger(): force=True) -# Initialize logger now because it will protect against -# handlers that get created when classifer and detector are intialized -logger() - - -# Set interval for checking in -CHECKIN_INTERVAL = CONFIG['checkin_interval'] -print("Loading classifier") -# load models once -CLASSIFIER_MODEL, CLASSES = load_model(CLASSIFIER, LABELS) -print("Finished loading classifier") -print("Begin loading detector") -DETECTOR_MODEL = megadetector.MegaDetector(DETECTOR) -print("Finished loading detector") - - def fetch_detect_alert(): '''Functions for fetching images, detection, and sending alerts''' # Run the scheduler @@ -104,7 +60,46 @@ def fetch_detect_alert(): def main(): - ''''Runs main program and schedules future runs''' + '''Runs main program and schedules future runs''' + logger() + # Numpy FutureWarnings from tensorflow import + warnings.filterwarnings('ignore', category=FutureWarning) + # Parse arguments + PARSER = argparse.ArgumentParser(description='Retrieves images from \ + email & web scraper & runs detection') + PARSER.add_argument('config', type=str, help='Path to config file') + ARGS = PARSER.parse_args() + CONFIG_FILE = ARGS.config + # Load Configuration Settings from YML file + with open(CONFIG_FILE, 'r', encoding='utf-8') as stream: + CONFIG = yaml.safe_load(stream) + with open("config/cameratraps.yml", 'r') as stream: + camera_traps_config = yaml.safe_load(stream) + sys.path.append(camera_traps_config['camera_traps_path']) + # Set Email Variables for fetching + USERNAME = CONFIG['username'] + PASSWORD = CONFIG['password'] + TOKEN = CONFIG['token'] + AUTH = CONFIG['authorization'] + CLASSIFIER = CONFIG['classifier_model'] + DETECTOR = CONFIG['detector_model'] + DEV_EMAILS = CONFIG['dev_emails'] + HOST = 'imap.gmail.com' + RUN_SCHEDULER = CONFIG['run_scheduler'] + VISUALIZE_OUTPUT = CONFIG['visualize_output'] + LABELS = CONFIG['classes'] + # Set interval for checking in + CHECKIN_INTERVAL = CONFIG['checkin_interval'] + + # Load models once + print("Loading classifier") + CLASSIFIER_MODEL, CLASSES = load_model(CLASSIFIER, LABELS) + print("Finished loading classifier") + + print("Begin loading detector") + DETECTOR_MODEL = megadetector.MegaDetector(DETECTOR) + print("Finished loading detector") + fetch_detect_alert() if VISUALIZE_OUTPUT is True: schedule.every(RUN_SCHEDULER).seconds.do(fetch_detect_alert) From 2f35a3e5e96635515e05a232d414221d39307cd1 Mon Sep 17 00:00:00 2001 From: Katie Garwood Date: Mon, 26 May 2025 16:46:53 -0700 Subject: [PATCH 4/5] move the extra random comment to the docstring there is important documentation about the different api calls you can do from strikeforce, but it should be clearly stored in the docstring and not in a floating comment in the middle --- cougarvision_utils/get_images.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/cougarvision_utils/get_images.py b/cougarvision_utils/get_images.py index 9ecee47..b318c62 100644 --- a/cougarvision_utils/get_images.py +++ b/cougarvision_utils/get_images.py @@ -10,20 +10,8 @@ last_id.txt as well, but it creates a new one if there is not one currently present. -''' - -import json -import urllib.request -import os.path -import os -import logging -import requests -import numpy as np -from cougarvision_visualize.visualize_helper import get_last_file_number - - -''' -request examples +Here are some different strikeforce api commands for future +reference: get list of camaras request <- "cameras" @@ -52,6 +40,15 @@ parameters <- "" ''' +import json +import urllib.request +import os.path +import os +import logging +import requests +import numpy as np +from cougarvision_visualize.visualize_helper import get_last_file_number + def request_strikeforce(username, auth_token, base, request, parameters): ''' From 86aa246071b8df51f250276d6cd2f134a7b17df1 Mon Sep 17 00:00:00 2001 From: Katie Garwood Date: Mon, 16 Jun 2025 16:24:17 -0700 Subject: [PATCH 5/5] fix variable passing issue when making functions before this whole script was not nested in functions at all. One of the things this pr does is make fetch and alert into functions, but we werent passing the correct variables to the functions but now we are --- fetch_and_alert.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fetch_and_alert.py b/fetch_and_alert.py index 5e4e6cf..ab2fe69 100644 --- a/fetch_and_alert.py +++ b/fetch_and_alert.py @@ -25,6 +25,7 @@ import logging import yaml import schedule +import sys from cougarvision_utils.detect_img import detect @@ -45,7 +46,7 @@ def logger(): force=True) -def fetch_detect_alert(): +def fetch_detect_alert(CONFIG, CLASSIFIER, CLASSES, DETECTOR): '''Functions for fetching images, detection, and sending alerts''' # Run the scheduler print("Running fetch_and_alert") @@ -54,7 +55,7 @@ def fetch_detect_alert(): print('Finished fetching images') print('Starting Detection') for i in images: - detect(i, CONFIG, CLASSIFIER_MODEL, CLASSES, DETECTOR_MODEL) + detect(i, CONFIG, CLASSIFIER, CLASSES, DETECTOR) print('Finished Detection') print("Sleeping since: " + str(dt.now())) @@ -81,8 +82,8 @@ def main(): PASSWORD = CONFIG['password'] TOKEN = CONFIG['token'] AUTH = CONFIG['authorization'] - CLASSIFIER = CONFIG['classifier_model'] - DETECTOR = CONFIG['detector_model'] + CLASSIFIER_MODEL = CONFIG['classifier_model'] + DETECTOR_MODEL = CONFIG['detector_model'] DEV_EMAILS = CONFIG['dev_emails'] HOST = 'imap.gmail.com' RUN_SCHEDULER = CONFIG['run_scheduler'] @@ -93,18 +94,18 @@ def main(): # Load models once print("Loading classifier") - CLASSIFIER_MODEL, CLASSES = load_model(CLASSIFIER, LABELS) + CLASSIFIER, CLASSES = load_model(CLASSIFIER_MODEL, LABELS) print("Finished loading classifier") print("Begin loading detector") - DETECTOR_MODEL = megadetector.MegaDetector(DETECTOR) + DETECTOR = megadetector.MegaDetector(DETECTOR_MODEL) print("Finished loading detector") - fetch_detect_alert() + fetch_detect_alert(CONFIG, CLASSIFIER, CLASSES, DETECTOR) if VISUALIZE_OUTPUT is True: - schedule.every(RUN_SCHEDULER).seconds.do(fetch_detect_alert) + schedule.every(RUN_SCHEDULER).seconds.do(fetch_detect_alert(CONFIG, CLASSIFIER, CLASSES, DETECTOR)) else: - schedule.every(RUN_SCHEDULER).minutes.do(fetch_detect_alert) + schedule.every(RUN_SCHEDULER).minutes.do(fetch_detect_alert(CONFIG, CLASSIFIER, CLASSES, DETECTOR)) schedule.every(CHECKIN_INTERVAL).hours.do(checkin, DEV_EMAILS, USERNAME, PASSWORD, HOST) # schedule.every(30).days.do(post_monthly_obs, TOKEN, AUTH)