diff --git a/learning_observer/learning_observer/auth/__init__.py b/learning_observer/learning_observer/auth/__init__.py index 5edc27b78..6aa68c941 100644 --- a/learning_observer/learning_observer/auth/__init__.py +++ b/learning_observer/learning_observer/auth/__init__.py @@ -44,6 +44,7 @@ We haven't figured out all of the data models here. ''' +import pmss import sys # Decorators to confirm requests are authenticated @@ -78,6 +79,34 @@ import learning_observer.prestartup import learning_observer.settings as settings +# HACK These enabled items ought to be implicitly defined by pmss if +# the child settings are available. For now, we define these manually +# in the creds.pmss file. +pmss.register_field( + name='google_oauth_enabled', + type=pmss.pmsstypes.TYPES.boolean, + description='Flag for determining if we should use Google OAuth or not.', + default=False +) +pmss.register_field( + name='password_file_enabled', + type=pmss.pmsstypes.TYPES.boolean, + description='Flag for determining if we should use a Password File or not.', + default=False +) +pmss.register_field( + name='http_basic_auth_enabled', + type=pmss.pmsstypes.TYPES.boolean, + description='Flag for determining if we should use a HTTP Basic or not.', + default=False +) + +pmss.register_field( + name='password_file', + type=pmss.pmsstypes.TYPES.string, + description='Path to the password file on the system.' +) + @learning_observer.prestartup.register_startup_check def verify_auth_precheck(): @@ -85,30 +114,22 @@ def verify_auth_precheck(): This is a pre-startup check to make sure that the auth system is configured correctly. ''' - # We need some auth - if 'auth' not in settings.settings: - raise learning_observer.prestartup.StartupCheck( - "Please configure auth") - # If we have Google oauth, we need it properly configured. # TODO: Confirm everything works with Google Oauth missing - if 'google_oauth' in settings.settings['auth']: - if 'web' not in settings.settings['auth']['google_oauth'] or \ - 'client_secret' not in settings.settings['auth']['google_oauth']['web'] or \ - 'project_id' not in settings.settings['auth']['google_oauth']['web'] or \ - 'client_id' not in settings.settings['auth']['google_oauth']['web'] or \ - isinstance(settings.settings['auth']['google_oauth']['web']['client_secret'], dict) or \ - isinstance(settings.settings['auth']['google_oauth']['web']['project_id'], dict) or \ - isinstance(settings.settings['auth']['google_oauth']['web']['client_id'], dict): - error = \ - "Please configure (or disable) Google oauth\n" + \ - "\n" + \ - "Go to:\n" + \ - " https://console.developers.google.com/ \n" + \ - "And set up an OAuth client for a web application. Make sure that configuration\n" + \ - "mirrors the one here.\n" + \ - "\n" + \ - "If you are not planning to use Google auth (which is the case for most dev\n" + \ - "settings), please disable Google authentication in creds.yaml by\n" + \ - "removing the google_auth section under auth." - raise learning_observer.prestartup.StartupCheck("Auth: " + error) + try: + settings.pmss_settings.client_secret(types=['auth', 'google_oauth', 'web']) + settings.pmss_settings.project_id(types=['auth', 'google_oauth', 'web']) + settings.pmss_settings.client_id(types=['auth', 'google_oauth', 'web']) + except Exception as e: + error = \ + "Please configure (or disable) Google oauth\n" + \ + "\n" + \ + "Go to:\n" + \ + " https://console.developers.google.com/ \n" + \ + "And set up an OAuth client for a web application. Make sure that configuration\n" + \ + "mirrors the one here.\n" + \ + "\n" + \ + "If you are not planning to use Google auth (which is the case for most dev\n" + \ + "settings), please disable Google authentication in creds.yaml by\n" + \ + "removing the google_auth section under auth." + raise learning_observer.prestartup.StartupCheck("Auth: " + error) diff --git a/learning_observer/learning_observer/auth/events.py b/learning_observer/learning_observer/auth/events.py index 96b7a60ec..8962afb52 100644 --- a/learning_observer/learning_observer/auth/events.py +++ b/learning_observer/learning_observer/auth/events.py @@ -20,10 +20,9 @@ Some of these code paths are untested. Please test and debug before using. ''' -import asyncio +import pmss import urllib.parse import secrets -import sys import aiohttp_session import aiohttp.web @@ -37,9 +36,22 @@ from learning_observer.log_event import debug_log +AVAILABLE_AUTH_METHODS = ['local_storage', 'chromebook', 'testcase_auth', 'guest'] AUTH_METHODS = {} +pmss.register_field( + name='userfile', + type=pmss.TYPES.string, + description='Filename for determining which users to authenticate.' +) +pmss.register_field( + name='allow_guest', + type=pmss.TYPES.boolean, + description='Flag for if guests are allowed for a given auth method.' +) + + def register_event_auth(name): ''' Decorator to register a method to authenticate events @@ -105,13 +117,13 @@ def token_authorize_user(auth_method, user_id_token): ''' Authorize a user based on a list of allowed user ID tokens ''' - am_settings = learning_observer.settings.settings['event_auth'][auth_method] - if 'userfile' in am_settings: - userfile = am_settings['userfile'] + userfile = learning_observer.settings.pmss_settings.userfile(types=['event_auth', auth_method]) + if userfile: users = [u.strip() for u in open(learning_observer.paths.data(userfile)).readlines()] if user_id_token in users: return "authenticated" - if am_settings.get("allow_guest", False): + allow_guest = learning_observer.settings.pmss_settings.allow_guest(types=['event_auth', auth_method]) + if allow_guest: return "unauthenticated" raise aiohttp.web.HTTPUnauthorized() @@ -311,7 +323,7 @@ async def authenticate(request, headers, first_event, source): 2. Providence: How they were authenticated (if at all), or how we believe they are who they are. 3. `user_id` -- a unique user identifier ''' - for auth_method in learning_observer.settings.settings['event_auth']: + for auth_method in AVAILABLE_AUTH_METHODS: auth_metadata = await AUTH_METHODS[auth_method](request, headers, first_event, source) if auth_metadata: if "safe_user_id" not in auth_metadata: @@ -330,9 +342,7 @@ def check_event_auth_config(): Check that all event auth methods are correctly configured, before events come in. ''' - if 'event_auth' not in learning_observer.settings.settings: - raise learning_observer.prestartup.StartupCheck("Please configure event authentication") - for auth_method in learning_observer.settings.settings['event_auth']: + for auth_method in AVAILABLE_AUTH_METHODS: if auth_method not in AUTH_METHODS: raise learning_observer.prestartup.StartupCheck( "Please configure event authentication for {}\n(Methods: {})".format( @@ -342,6 +352,7 @@ def check_event_auth_config(): if __name__ == "__main__": + import asyncio import doctest print("Running tests") diff --git a/learning_observer/learning_observer/auth/handlers.py b/learning_observer/learning_observer/auth/handlers.py index 4506afdaa..5d115ca87 100644 --- a/learning_observer/learning_observer/auth/handlers.py +++ b/learning_observer/learning_observer/auth/handlers.py @@ -6,6 +6,7 @@ import base64 import json +import pmss import random import aiohttp @@ -21,6 +22,22 @@ import names +# TODO we ought to define all the auth items for pmss +# in one spot. Currently they are split up across +# multiple files. +pmss.register_field( + name='test_case_insecure', + type=pmss.TYPES.boolean, + description='For testing to allow no-login required.', + default=False +) +pmss.register_field( + name='demo_insecure', + type=pmss.TYPES.string, + description='Similar to `test_case_insecure`, but provides a name.', + default=None +) + async def logout_handler(request): """ @@ -67,7 +84,7 @@ async def test_case_user(request): This is a short circuit for test cases without logging in. THIS SHOULD NEVER BE ENABLED ON A LIVE SERVER ''' - tci = learning_observer.settings.settings['auth'].get("test_case_insecure", False) + tci = learning_observer.settings.pmss_settings.test_case_insecure(types=['auth']) if not tci: return None if not isinstance(tci, dict): @@ -96,7 +113,7 @@ async def demo_user(request): In contrast to the test case user, this assigns a dummy name and similar. That's bad for testing, where we want determinism, but it's good for demos. ''' - if not learning_observer.settings.settings['auth'].get("demo_insecure", False): + if not learning_observer.settings.pmss_settings.demo_insecure(types=['auth']): return None def name_to_email(name): @@ -115,9 +132,9 @@ def name_to_email(name): name = name.split() return name[0][0].lower() + name[-1].lower() + "@localhost" - demo_auth_setting = learning_observer.settings.settings['auth']["demo_insecure"] - if isinstance(demo_auth_setting, dict) and 'name' in demo_auth_setting: - name = demo_auth_setting['name'] + demo_auth_setting = learning_observer.settings.pmss_settings.demo_insecure(types=['auth']) + if demo_auth_setting: + name = demo_auth_setting else: name = names.get_full_name() diff --git a/learning_observer/learning_observer/auth/http_basic.py b/learning_observer/learning_observer/auth/http_basic.py index 433e8d979..d29494e7c 100644 --- a/learning_observer/learning_observer/auth/http_basic.py +++ b/learning_observer/learning_observer/auth/http_basic.py @@ -16,6 +16,7 @@ ''' import base64 import json +import pmss import yaml import sys @@ -30,6 +31,29 @@ from learning_observer.log_event import debug_log +# TODO I noticed that this code consistently referred to +# `http_basic` whereas the creds example that I've been using +# refers to `http_basic_auth`. We ought make sure we are +# using a consistent reference. +pmss.register_field( + name='full_site_auth', + type=pmss.TYPES.boolean, + description='', # TODO + default=False +) +pmss.register_field( + name='login_page_enabled', + type=pmss.TYPES.boolean, + description='', # TODO + default=False +) +pmss.register_field( + name='delegate_nginx_auth', + type=pmss.TYPES.boolean, + description='', # TODO, + default=False +) + def http_basic_extract_username_password(request): ''' @@ -73,10 +97,9 @@ def http_auth_middleware_enabled(): to accidentally receive requests with auth headers on pages which nginx has not secured. ''' - if 'http_basic' not in learning_observer.settings.settings['auth']: + if not learning_observer.settings.pmss_settings.http_basic_auth_enabled(types=['auth']): return False - auth_basic_settings = learning_observer.settings.settings['auth']['http_basic'] - return auth_basic_settings.get("full_site_auth", False) + return learning_observer.settings.pmss_settings.full_site_auth(types=['auth', 'http_basic_auth']) def http_auth_page_enabled(): @@ -88,11 +111,10 @@ def http_auth_page_enabled(): this. ''' # Is http basic auth enabled? - if 'http_basic' not in learning_observer.settings.settings['auth']: + if not learning_observer.settings.pmss_settings.http_basic_auth_enabled(types=['auth']): return False - auth_basic_settings = learning_observer.settings.settings['auth']['http_basic'] # And is it configured with a dedicated login page? - if not auth_basic_settings.get("login_page_enabled", False): + if not learning_observer.settings.pmss_settings.login_page_enabled(types=['auth', 'http_basic_auth']): return False return True @@ -185,9 +207,9 @@ def http_basic_startup_check(): ) if ( - 'http_basic' in learning_observer.settings.settings['auth'] - and learning_observer.settings.settings['auth']['http_basic'].get("delegate_nginx_auth", False) - and learning_observer.settings.settings['auth']['http_basic'].get("password_file", False) + learning_observer.settings.pmss_settings.http_basic_auth_enabled(types=['auth']) + and learning_observer.settings.pmss_settings.password_file(types=['auth', 'http_basic_auth']) + and learning_observer.settings.pmss_settings.delegate_nginx_auth(types=['auth', 'http_basic_auth']) ): raise learning_observer.prestartup.StartupCheck( "Your HTTP Basic authentication is misconfigured.\n" diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 0bad8f1b7..060242fe6 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -78,6 +78,12 @@ description="The Google OAuth client secret", required=True ) +pmss.register_field( + name="project_id", + type=pmss.pmsstypes.TYPES.string, + description="The Google OAuth project id", + required=True +) pmss.register_field( name='fetch_additional_info_from_teacher_on_login', type=pmss.pmsstypes.TYPES.boolean, @@ -106,6 +112,9 @@ ] # TODO Type list is not yet supported by PMSS 4/24/24 +# We did another pass of PMSS conversion and lists are still +# not supported. Our current strategy is to try and make things +# work without lists until they are supported 2/10/24 # pmss.register_field( # name='base_scopes', # type='list', diff --git a/learning_observer/learning_observer/auth/utils.py b/learning_observer/learning_observer/auth/utils.py index f06bb6816..3795c95d5 100644 --- a/learning_observer/learning_observer/auth/utils.py +++ b/learning_observer/learning_observer/auth/utils.py @@ -178,7 +178,7 @@ def _role_required(role): def decorator(func): @functools.wraps(func) async def wrapper(request): - if learning_observer.settings.settings['auth'].get("test_case_insecure", False): + if learning_observer.settings.pmss_settings.test_case_insecure(types=['auth']): return await func(request) '''TODO evaluate how we should be using `role` with the `authorized` key. diff --git a/learning_observer/learning_observer/client_config.py b/learning_observer/learning_observer/client_config.py index 4b1c0d20e..f7942ad42 100644 --- a/learning_observer/learning_observer/client_config.py +++ b/learning_observer/learning_observer/client_config.py @@ -11,10 +11,37 @@ ''' import aiohttp +import pmss import learning_observer.settings import learning_observer.auth.http_basic +# TODO where should we define the theme items? +pmss.register_field( + name='server_name', + type=pmss.pmsstypes.TYPES.string, + description='Name of the server to include on the login page.', + default='Learning Observer' +) +pmss.register_field( + name='front_page_pitch', + type=pmss.pmsstypes.TYPES.string, + description='Short server description to include on the login page.' +) +pmss.register_field( + name='logo_big', + type=pmss.pmsstypes.TYPES.string, + description='Path to logo to display on the login page.' +) +# This item is used in routes, but we are leaving it here +# along with the rest of the `theme` items. +pmss.register_field( + name='root_file', + type=pmss.pmsstypes.TYPES.string, + description="We'd like to be able to have the root page themeable, for non-ETS deployments. This is a quick-and-dirty way to override the main page.", + default='webapp.html' +) + async def client_config_handler(request): ''' @@ -34,10 +61,14 @@ async def client_config_handler(request): 'hide_labels': False # TODO: Should be loaded from config file } }, - "google_oauth": "google_oauth" in learning_observer.settings.settings['auth'], - "password_auth": "password_file" in learning_observer.settings.settings['auth'], + "google_oauth": learning_observer.settings.pmss_settings.google_oauth_enabled(types=['auth']), + "password_auth": learning_observer.settings.pmss_settings.password_file_enabled(types=['auth']), "http_basic_auth": learning_observer.auth.http_basic.http_auth_page_enabled(), - "theme": learning_observer.settings.settings['theme'] + "theme": { + 'server_name': learning_observer.settings.pmss_settings.server_name(types=['theme']), + 'front_page_pitch': learning_observer.settings.pmss_settings.front_page_pitch(types=['theme']), + 'logo_big': learning_observer.settings.pmss_settings.logo_big(types=['theme']), + } } return aiohttp.web.json_response(client_config) diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index 93c87f8c7..4750d2bb3 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -183,6 +183,9 @@ def connect_to_google_cache(): for key in ['save_google_ajax', 'use_google_ajax', 'save_clean_ajax', 'use_clean_ajax']: if settings.feature_flag(key): global cache + kvs_router = learning_observer.kvs.initialize_kvs() + google_kvs_class = learning_observer.kvs.prepare_kvs('google_cache') + kvs_router.add_item('google_cache', google_kvs_class) try: cache = learning_observer.kvs.KVS.google_cache() except AttributeError: diff --git a/learning_observer/learning_observer/kvs.py b/learning_observer/learning_observer/kvs.py index 469761b7f..91997cd9f 100644 --- a/learning_observer/learning_observer/kvs.py +++ b/learning_observer/learning_observer/kvs.py @@ -18,6 +18,7 @@ import json import os import os.path +import pmss import learning_observer.paths import learning_observer.prestartup @@ -27,6 +28,28 @@ OBJECT_STORE = dict() +pmss.register_field( + name='type', + type=pmss.TYPES.string, # TODO one of + description='The type of KVS being defined' +) +pmss.register_field( + name='expiry', + type=pmss.TYPES.integer, + description='How long the data lasts within a KVS.' +) +pmss.register_field( + name='path', + type=pmss.TYPES.string, + description='The path on to the directory on the filesystem to use as a KVS.' +) +pmss.register_field( + name='subdirs', + type=pmss.TYPES.boolean, + description='If set, keys with slashes will result in the creation of subdirs', + default=False +) + class _KVS: async def dump(self, filename=None): @@ -311,10 +334,38 @@ def __init__(self, key, type, param): super().__init__(msg) +def prepare_kvs(key): + kvs_type = learning_observer.settings.pmss_settings.type(types=['kvs', key]) + if kvs_type not in KVS_MAP: + raise KeyError(f"Invalid KVS type '{kvs_type}'") + kvs_class = KVS_MAP[kvs_type] + if kvs_type == 'redis_ephemeral': + try: + expiry = learning_observer.settings.pmss_settings.expiry(types=['kvs', key]) + except: + raise MissingKVSParameters(key, kvs_type, 'expiry') + kvs_class = functools.partial(kvs_class, expiry) + elif kvs_type == 'filesystem': + try: + filesystem_path = learning_observer.settings.pmss_settings.path(types=['kvs', key]) + allow_subdirs = learning_observer.settings.pmss_settings.subdirs(types=['kvs', key]) + except: + raise MissingKVSParameters(key, kvs_type, 'path') + kvs_class = functools.partial(kvs_class, filesystem_path, allow_subdirs) + return kvs_class + + +def initialize_kvs(): + global KVS + if KVS is None: + KVS = KVSRouter() + return KVS + + class KVSRouter: """Routes KVS calls to the appropriate backend.""" - def __init__(self, default=None, items=None): + def __init__(self, default=None): """ Initialize the router with a default KVS and optional additional items. @@ -323,21 +374,10 @@ def __init__(self, default=None, items=None): """ self._items = {'default': default} - if items is not None: - for key, kvs_item in items: - kvs_type = kvs_item['type'] - if kvs_type not in KVS_MAP: - raise KeyError(f"Invalid KVS type '{kvs_type}'") - kvs_class = KVS_MAP[kvs_type] - if kvs_type == 'redis_ephemeral': - if 'expiry' not in kvs_item: - raise MissingKVSParameters(key, kvs_type, 'expiry') - kvs_class = functools.partial(kvs_class, kvs_item['expiry']) - elif kvs_type == 'filesystem': - if 'path' not in kvs_item: - raise MissingKVSParameters(key, kvs_type, 'path') - kvs_class = functools.partial(kvs_class, kvs_item['path'], kvs_item.get('subdirs', False)) - self.add_item(key, kvs_class) + general_use_kvs = ['default', 'memoization'] + for key in general_use_kvs: + kvs_class = prepare_kvs(key) + self.add_item(key, kvs_class) def __call__(self): """Call the default KVS callable.""" @@ -371,35 +411,7 @@ def kvs_startup_check(): Checks like this one allow us to fail on startup, rather than later ''' - global KVS - try: - KVS = KVSRouter(items=learning_observer.settings.settings['kvs'].items()) - - except KeyError: - if 'kvs' not in learning_observer.settings.settings: - raise learning_observer.prestartup.StartupCheck( - "No KVS configured. Please set kvs.type in settings.py\n" - "Look at example settings file to see what's available." - ) - elif any([kvs['type'] not in KVS_MAP for _, kvs in learning_observer.settings.settings['kvs'].items()]): - raise learning_observer.prestartup.StartupCheck( - "Unknown KVS type: {}\n" - "Look at example settings file to see what's available. \n" - "Suppported types: {}".format( - ', '.join([kvs['type'] for _, kvs in learning_observer.settings.settings['kvs'].items() if kvs['type'] not in KVS_MAP]), - list(KVS_MAP.keys()) - ) - ) - elif 'default' not in learning_observer.settings.settings['kvs']: - raise learning_observer.prestartup.StartupCheck( - "No default KVS configured. Please set default kvs.\n" - "See example settings file for usage." - ) - else: - raise learning_observer.prestartup.StartupCheck( - "KVS incorrectly configured. Please fix the error, and\n" - "then replace this with a more meaningful error message" - ) + initialize_kvs() return True diff --git a/learning_observer/learning_observer/log_event.py b/learning_observer/learning_observer/log_event.py index 4f16b686c..5408f2395 100644 --- a/learning_observer/learning_observer/log_event.py +++ b/learning_observer/learning_observer/log_event.py @@ -112,7 +112,8 @@ class LogLevel(Enum): description='How much information do we want to log.\n'\ '`NONE`: do not print anything\n'\ '`SIMPLE`: print simple debug messages\n'\ - '`EXTENDED`: print debug message with stack trace and timestamp' + '`EXTENDED`: print debug message with stack trace and timestamp', + default=LogLevel.NONE.value ) class LogDestination(Enum): @@ -123,6 +124,19 @@ class LogDestination(Enum): FILE = 'FILE' +# TODO eventually we want to support a list of debug destinations +# however pmss does not support lists at this time. Instead we +# only allow a single debug destination to be defined in settings. +pmss.parser('debug_log_destinations', parent='string', choices=[level.value for level in LogDestination], transform=None) +pmss.register_field( + name='debug_log_destinations', + type='debug_log_destinations', + description='Where do we want to log the information.\n'\ + '`CONSOLE`: print debug information to the console\n'\ + '`FILE`: print debug information to a file', + default=LogDestination.CONSOLE.value +) + # Before we've read the settings file, we'll log basic messages to the # console and to the log file. DEBUG_LOG_LEVEL = LogLevel.SIMPLE @@ -152,11 +166,8 @@ def initialize_logging_framework(): DEBUG_LOG_DESTINATIONS = [LogDestination.CONSOLE, LogDestination.FILE] # In either case, we want to override from the settings file. - if "logging" in settings.settings: - if "debug_log_level" in settings.settings["logging"]: - DEBUG_LOG_LEVEL = LogLevel(settings.pmss_settings.debug_log_level(types=['logging'])) - if "debug_log_destinations" in settings.settings["logging"]: - DEBUG_LOG_DESTINATIONS = list(map(LogDestination, settings.settings["logging"]["debug_log_destinations"])) + DEBUG_LOG_LEVEL = LogLevel(settings.pmss_settings.debug_log_level(types=['logging'])) + DEBUG_LOG_DESTINATIONS = [LogDestination(settings.pmss_settings.debug_log_destinations(types=['logging']))] debug_log("DEBUG_LOG_LEVEL:", DEBUG_LOG_LEVEL) debug_log("DEBUG_DESTINATIONS:", DEBUG_LOG_DESTINATIONS) diff --git a/learning_observer/learning_observer/paths.py b/learning_observer/learning_observer/paths.py index 2dc505ca1..bdc4fb1b7 100644 --- a/learning_observer/learning_observer/paths.py +++ b/learning_observer/learning_observer/paths.py @@ -55,7 +55,7 @@ def config_file(): ''' Main configuration file ''' - pathname = os.path.join(os.path.dirname(base_path()), 'creds.yaml') + pathname = os.path.join(os.path.dirname(base_path()), 'creds.pmss') # TODO: This is cut-and-paste from settings.py. # # It should be one place. diff --git a/learning_observer/learning_observer/prestartup.py b/learning_observer/learning_observer/prestartup.py index 54debdb15..d221a771b 100644 --- a/learning_observer/learning_observer/prestartup.py +++ b/learning_observer/learning_observer/prestartup.py @@ -284,10 +284,12 @@ def startup_checks_and_init(): @register_startup_check def check_aio_session_settings(): - if 'aio' not in settings.settings or \ - 'session_secret' not in settings.settings['aio'] or \ - isinstance(settings.settings['aio']['session_secret'], dict) or \ - 'session_max_age' not in settings.settings['aio']: + try: + # Attempt to fetch settings, if an exception is raised, the + # settings are not configured. + settings.pmss_settings.session_max_age(types=['aio']) + settings.pmss_settings.session_secret(types=['aio']) + except: raise StartupCheck( "Settings file needs an `aio` section with a `session_secret`\n" "subsection containing a secret string. This is used for\n" diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 4329eb5bf..1fb6b8809 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -348,17 +348,13 @@ def init(): or smaller functions otherwise. ''' global ajax - roster_source = settings.pmss_settings.source(types=['roster_data']) - if 'roster_data' not in settings.settings: - print(settings.settings) - raise learning_observer.prestartup.StartupCheck( - "Settings file needs a `roster_data` element with a `source` element. No `roster_data` element found." - ) - elif 'source' not in settings.settings['roster_data']: + try: + roster_source = settings.pmss_settings.source(types=['roster_data']) + except: raise learning_observer.prestartup.StartupCheck( - "Settings file needs a `roster_data` element with a `source` element. No `source` element found." + "Settings file needs a `roster_data` element with a `source` element. No relevant element found." ) - elif roster_source in ['test', 'filesystem']: + if roster_source in ['test', 'filesystem']: ajax = synthetic_ajax elif roster_source in ["google_api"]: ajax = google_ajax diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index 843be52ab..457461e66 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -4,6 +4,7 @@ import getpass import os +import pmss import secrets import sys @@ -33,6 +34,15 @@ from learning_observer.utility_handlers import * +# NOTE this was previously listed under config.debug where debug +# was a list of nothing or only tracemalloc. +pmss.register_field( + name='tracemalloc_enabled', + type=pmss.pmsstypes.TYPES.boolean, + description='Flag for determining if we should include trace memory allocations.', + default=False +) + def add_routes(app): ''' @@ -43,7 +53,7 @@ def add_routes(app): ''' # Allow debugging of memory leaks. Helpful, but this is a massive # resource hog. Don't accidentally turn this on in prod :) - if 'tracemalloc' in settings.settings['config'].get("debug", []): + if settings.pmss_settings.tracemalloc_enabled(types=['config']): import tracemalloc tracemalloc.start(25) @@ -115,7 +125,7 @@ def tracemalloc_handler(request): # We'd like to be able to have the root page themeable, for # non-ETS deployments. This is a quick-and-dirty way to override # the main page. - root_file = settings.settings.get("theme", {}).get("root_file", "webapp.html") + root_file = settings.pmss_settings.root_file(types=['theme']) app.add_routes([ aiohttp.web.get('/', static_file_handler(paths.static(root_file))), ]) @@ -236,7 +246,7 @@ def register_auth_webapp_views(app): handler=learning_observer.auth.user_info_handler) ]) - if 'google_oauth' in settings.settings['auth']: + if settings.pmss_settings.google_oauth_enabled(types=['auth']): debug_log("Running with Google authentication") app.add_routes([ aiohttp.web.get( @@ -244,16 +254,17 @@ def register_auth_webapp_views(app): handler=learning_observer.auth.social_handler), ]) - if 'password_file' in settings.settings['auth']: + if settings.pmss_settings.password_file_enabled(types=['auth']): debug_log("Running with password authentication") - if not os.path.exists(settings.settings['auth']['password_file']): + password_file = settings.pmss_settings.password_file(types=['auth']) + if not os.path.exists(password_file): print("Configured to run with password file," "but no password file exists") print() print("Please either:") print("* Remove auth/password_file from the settings file") print("* Create a file {fn} with lo_passwd.py".format( - fn=settings.settings['auth']['password_file'] + fn=password_file )) print("Typically:") print("{python_src} learning_observer/util/lo_passwd.py " @@ -262,14 +273,14 @@ def register_auth_webapp_views(app): python_src=paths.PYTHON_EXECUTABLE, username=getpass.getuser(), password=secrets.token_urlsafe(16), - fn=settings.settings['auth']['password_file'] + fn=password_file )) sys.exit(-1) app.add_routes([ aiohttp.web.post( '/auth/login/password', learning_observer.auth.password_auth( - settings.settings['auth']['password_file']) + password_file) )]) # If we want to support multiple modes of authentication, including @@ -281,7 +292,7 @@ def register_auth_webapp_views(app): # At the very least, the user should explicitly set it to `null` # if they are planning on using nginx for auth debug_log("Enabling http basic auth page") - auth_file = settings.settings['auth']['http_basic']["password_file"] + auth_file = settings.pmss_settings.password_file(types=['auth', 'http_basic']) app.add_routes([ aiohttp.web.get( '/auth/login/http-basic', diff --git a/modules/wo_bulk_essay_analysis/wo_bulk_essay_analysis/gpt.py b/modules/wo_bulk_essay_analysis/wo_bulk_essay_analysis/gpt.py index baf06a6a1..b59dbcb37 100644 --- a/modules/wo_bulk_essay_analysis/wo_bulk_essay_analysis/gpt.py +++ b/modules/wo_bulk_essay_analysis/wo_bulk_essay_analysis/gpt.py @@ -1,6 +1,7 @@ import aiohttp import loremipsum import os +import pmss import learning_observer.communication_protocol.integration from learning_observer.log_event import debug_log @@ -11,6 +12,21 @@ rubric_template = """{task}\n\n[Rubric]\n{rubric}""" gpt_responder = None +pmss.register_field( + name='model', + type=pmss.TYPES.string, + description='Which model we wish to use with the GPT responder.' +) +pmss.register_field( + name='api_key', + type=pmss.TYPES.string, + description='API Key required for connection.' +) +pmss.register_field( + name='host', + type=pmss.TYPES.string, # TODO this ought to be a hostname, but sometimes we want `None` + description='Where to connect for the GPT usage.' +) class GPTAPI: def chat_completion(self, prompt, system_prompt): @@ -143,33 +159,19 @@ def initialize_gpt_responder(): try the next one. ''' global gpt_responder - # TODO change this to use settings.module_settings() instead - # that method now uses pmss which doesn't support lists and - # dictionaries yet. - responders = learning_observer.settings.settings['modules']['writing_observer'].get('gpt_responders', {}) - exceptions = [] - for key in responders: - if key not in GPT_RESPONDERS: - exceptions.append(KeyError( - f'GPT Responder `{key}` is not yet configured on this system.\n'\ - f'The available responders are [{", ".join(GPT_RESPONDERS.keys())}].' - )) - continue - try: - gpt_responder = GPT_RESPONDERS[key](**responders[key]) - debug_log(f'INFO:: Using GPT responder `{key}` with model `{responders[key]["model"]}`') - return True - except GPTInitializationError as e: - exceptions.append(e) - debug_log(f'WARNING:: Unable to initialize GPT responder `{key}:`.\n{e}') - gpt_responder = None - no_responders = 'No GPT responders found in `creds.yaml`. To add a responder, add either'\ - '`openai` or `ollama` along with any subsettings to `modules.writing_observer.gpt_responders`.\n'\ - 'Example:\n```\ngpt_responders:\n ollama:\n model: llama2\n```' - exception_strings = '\n'.join(str(e) for e in exceptions) if len(exceptions) > 0 else no_responders - exception_text = 'Unable to initialize a GPT responder. Encountered the following errors:\n'\ - f'{exception_strings}' - raise learning_observer.prestartup.StartupCheck("GPT: " + exception_text) + responder_type = learning_observer.settings.pmss_settings.type(types=['writing_observer', 'gpt_responder']) + if responder_type not in GPT_RESPONDERS: + raise KeyError( + f'GPT Responder `{key}` is not yet configured on this system.\n'\ + f'The available responders are [{", ".join(GPT_RESPONDERS.keys())}].' + ) + responder_kwargs = { + 'model': learning_observer.settings.pmss_settings.model(types=['writing_observer', 'gpt_responder']), + 'host': learning_observer.settings.pmss_settings.host(types=['writing_observer', 'gpt_responder']), + 'api_key': learning_observer.settings.pmss_settings.api_key(types=['writing_observer', 'gpt_responder']) + } + gpt_responder = GPT_RESPONDERS[responder_type](**responder_kwargs) + debug_log(f'INFO:: Using GPT responder `{responder_type}` with model `{responder_kwargs["model"]}`') @learning_observer.communication_protocol.integration.publish_function('wo_bulk_essay_analysis.gpt_essay_prompt') @@ -211,7 +213,7 @@ async def gpt(gpt_prompt): async def test_responder(): - responder = OllamaGPT() + responder = OllamaGPT(model='llama2', host=None) response = await responder.chat_completion('Why is the sky blue?', 'You are a helper agent, please help fulfill user requests.') print('Response:', response)