From b1ccec1bbb139c760a1df227a0c466fad96007d9 Mon Sep 17 00:00:00 2001 From: mheyen Date: Fri, 19 Feb 2021 13:35:04 +0100 Subject: [PATCH 01/79] Initial commit for multi-tenant-configuration - created initial structure - added main script - added config file - added environment folder - added configurations folder - added requirements.txt --- multi-tenant-configuration/README.md | 1 + multi-tenant-configuration/config.py | 19 +++ .../configurations/group_configuration.json | 113 ++++++++++++++++++ .../staging/opencast-organizations.yml | 74 ++++++++++++ multi-tenant-configuration/main.py | 82 +++++++++++++ multi-tenant-configuration/requirements.txt | 8 ++ 6 files changed, 297 insertions(+) create mode 100644 multi-tenant-configuration/README.md create mode 100644 multi-tenant-configuration/config.py create mode 100644 multi-tenant-configuration/configurations/group_configuration.json create mode 100644 multi-tenant-configuration/environment/staging/opencast-organizations.yml create mode 100644 multi-tenant-configuration/main.py create mode 100644 multi-tenant-configuration/requirements.txt diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md new file mode 100644 index 0000000..5b24ac0 --- /dev/null +++ b/multi-tenant-configuration/README.md @@ -0,0 +1 @@ +# Multi-tenants User configuration scripts for Opencast diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py new file mode 100644 index 0000000..3d15509 --- /dev/null +++ b/multi-tenant-configuration/config.py @@ -0,0 +1,19 @@ +# Configuration + +#Set this to your global admin node +url = "http://tenant1:8080" +#If you have multiple tenants use something like +#url_pattern = "https://{}.example.org" +#otherwise, url_pattern should be the same as the url variable above +url_pattern = "http://{}:8080" + +# digest user +digest_user = "opencast_system_account" +digest_pw = "CHANGE_ME" + +# path to environment configuration file +env_path = "environment/staging/opencast-organizations.yml" + +# workflow_definitions = ["import", "fast"] +# exclude_tenants = [] +# export_dir = "." \ No newline at end of file diff --git a/multi-tenant-configuration/configurations/group_configuration.json b/multi-tenant-configuration/configurations/group_configuration.json new file mode 100644 index 0000000..a27ceb1 --- /dev/null +++ b/multi-tenant-configuration/configurations/group_configuration.json @@ -0,0 +1,113 @@ +{ + "groups" : [ + { + "name": "System Administrators", + "description": "System Administrators", + "tenants": "all", + "type": "closed", + "members": [ + { + "name": "Guy 1", + "email": "test@test.de", + "reason": "Operations partner", + "uid": "guy-1", + "tenants": "all" + }, + { + "name": "Guy 2", + "email": "test@test.de", + "reason": "Operations partner", + "uid": "guy-2", + "tenants": "tenant1" + } + ], + "inactive_members": [ ], + "permissions": [ + { + "tenants": "all", + "roles": ["ROLE_ADMIN", "ROLE_SUDO"] + } + ] + }, + { + "name": "Organization Administrators", + "description": "Organization administrators have full access to all content of ${name}", + "tenants": "all", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI", + "ROLE_ORG_ADMIN" + ] + }, + { + "tenants" : "tenant2", + "roles": { + "add": [ + "ROLE_UI_EVENTS_DETAILS_ACL_VIEW", + "ROLE_UI_EVENTS_DETAILS_ACL_EDIT" + ], + "remove": [] + } + } + ] + }, + { + "name": "Producers", + "description": "Producers have limited access to content and functionality", + "tenants": "all", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI", + "ROLE_UI_EVENTS_CREATE" + ] + }, + { + "tenants" : "tenant1", + "roles": { + "add": [ + "ROLE_UI_EVENTS_COUNTERS_VIEW" + ], + "remove": [ + "ROLE_UI_EVENTS_CREATE" + ] + } + }, + { + "tenants" : "tenant2", + "roles": { + "add": [ + "ROLE_ORG_ADMIN" + ], + "remove": [] + } + } + ] + }, + { + "name": "Tenant2 Producers", + "description": "Tenant2 Producers have limited access to content and functionality", + "tenants": "tenant2", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI" + ] + } + ] + } + ] +} diff --git a/multi-tenant-configuration/environment/staging/opencast-organizations.yml b/multi-tenant-configuration/environment/staging/opencast-organizations.yml new file mode 100644 index 0000000..25320fd --- /dev/null +++ b/multi-tenant-configuration/environment/staging/opencast-organizations.yml @@ -0,0 +1,74 @@ +--- + +opencast_organizations: + - id: dummy + name: Dummy Tenant + aai_org: switch.ch + stream_sec_key: 5387689 + acl_default_template: organization + acl_default_download: False + acl_default_annotate: False + + # Global External API user passwords + opencast_system_account: + username: opencast_system_account + password: CHANGE_ME + switchcast_system_accounts: + - username: player + name: Player System User + email: test@test.de + password: 34dchG6nbhmhnG + roles: [ROLE_ADMIN, ROLE_SUDO] + - username: annotate + name: Annotate System User + email: test@test.de + password: jhvhuJH7utghfgfgJH + roles: [ROLE_ADMIN, ROLE_SUDO] + - username: cast + name: Cast System User + email: test@test.de + password: jhvhuJH7utghfgfgJH + roles: [ROLE_ADMIN, ROLE_SUDO] + capture_agent_accounts: [] + + - id: tenant1 + name: Tenant1 + aai_org: tenant1.ch + stream_sec_key: tu7uzgjjhghjf + capture_agent_accounts: + - username: ca-tenant1-ch + password: jvblkajklvjhaklehr + external_api_accounts: + - username: moodle-tenant1-ch + password: hghghjghdghdjd76 + name: Moodle System User + email: test@test.de + roles: [ROLE_EXTERNAL_APPLICATION] + - username: guy1 + password: abc + name: Guy 1 + email: test@test.de + roles: [ROLE_ADMIN] + - username: guy2 + password: abc + name: Guy 2 + email: test@test.de + roles: [ROLE_ADMIN, ROLE_SUDO] + - id: tenant2 + name: Tenant2 + aai_org: tenant2.ch + stream_sec_key: tu7uzgjjhghjf + capture_agent_accounts: + - username: ca-tenant2-ch + password: hjfkhfzuruzf76 + external_api_accounts: + - username: moodle-tenant2-ch + password: 67rdghn + name: Moodle System User + email: test@test.de + roles: [ROLE_EXTERNAL_APPLICATION] + - username: guy-1 + password: abc + name: Guy 1 + email: test@test.de + roles: [ROLE_ADMIN, ROLE_SUDO] diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py new file mode 100644 index 0000000..effd4ea --- /dev/null +++ b/multi-tenant-configuration/main.py @@ -0,0 +1,82 @@ +import os +import sys +import yaml + +sys.path.append(os.path.join(os.path.abspath('..'), "lib")) + +# import datetime +import config +import io +# from collections import defaultdict +from rest_requests.request_error import RequestError +from args.digest_login import DigestLogin +from rest_requests.request import get_request, post_request +# from pathlib import Path + + +# ToDo +# add logger +# add interaction question +# add parameter to python command + +def main(): + """ + configure Groups and Users + """ + + digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) + # read config file + opencast_organizations = read_configuration_file(config.env_path)['opencast_organizations'] + tenants = [tenant['id'] for tenant in opencast_organizations] + external_api_accounts = opencast_organizations[1]['external_api_accounts'] + + # create users for tenant 1 + for account in external_api_accounts: + tenant = tenants[1] + create_user(tenant, account, digest_login) + + +def read_configuration_file(path): + with open(path, 'r') as f: + conf = yaml.load(f, Loader=yaml.FullLoader) + + return conf + +# # example get request +# response = get_request("http://tenant1:8080/users/users.json", digest_login, "users/users.json") +# json_content = get_json_content(response) +# print(response) + +def get_roles_as_Json_array(account): + roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + + return roles + +def create_user(tenantid, account, digest_login): + """ sends a POST request to the admin UI to create a User + + :param tenantid: str tenant id to form correct url (e.g. 'tenant1') + :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} + :param digest_login: digest login + :return: + """ + url = '{}/admin-ng/users/'.format(config.url_pattern.format(tenantid)) + data = { + 'username': account['username'], + 'password': account['password'], + 'name': account['name'], + 'email': account['email'], + 'roles': str(get_roles_as_Json_array(account)) + } + # ToDo error handling + response = post_request(url, digest_login, '/admin-ng/users/', data=data) + + return response + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nAborting process.") + sys.exit(0) \ No newline at end of file diff --git a/multi-tenant-configuration/requirements.txt b/multi-tenant-configuration/requirements.txt new file mode 100644 index 0000000..e22b389 --- /dev/null +++ b/multi-tenant-configuration/requirements.txt @@ -0,0 +1,8 @@ +certifi==2020.11.8 +chardet==3.0.4 +idna==2.10 +requests==2.25.0 +requests-toolbelt==0.9.1 +urllib3==1.26.2 + +pyyaml \ No newline at end of file From a597f70ea2833123adf4864e76eed6bb8783d1a9 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 09:31:01 +0100 Subject: [PATCH 02/79] Nitpicking and minor corrections --- multi-tenant-configuration/config.py | 9 +++++---- multi-tenant-configuration/main.py | 2 +- multi-tenant-configuration/requirements.txt | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 3d15509..9307f0e 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -6,6 +6,11 @@ #url_pattern = "https://{}.example.org" #otherwise, url_pattern should be the same as the url variable above url_pattern = "http://{}:8080" +# list of tenant URLs +tenant_urls = [ + 'http://tenant1:8080', + 'http://tenant2:8080' +] # digest user digest_user = "opencast_system_account" @@ -13,7 +18,3 @@ # path to environment configuration file env_path = "environment/staging/opencast-organizations.yml" - -# workflow_definitions = ["import", "fast"] -# exclude_tenants = [] -# export_dir = "." \ No newline at end of file diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index effd4ea..75e3866 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -79,4 +79,4 @@ def create_user(tenantid, account, digest_login): main() except KeyboardInterrupt: print("\nAborting process.") - sys.exit(0) \ No newline at end of file + sys.exit(0) diff --git a/multi-tenant-configuration/requirements.txt b/multi-tenant-configuration/requirements.txt index e22b389..ba9cec7 100644 --- a/multi-tenant-configuration/requirements.txt +++ b/multi-tenant-configuration/requirements.txt @@ -5,4 +5,4 @@ requests==2.25.0 requests-toolbelt==0.9.1 urllib3==1.26.2 -pyyaml \ No newline at end of file +pyyaml==5.3.1 From 0c803f6274959dbddcb0f403118cb01c2931cbd7 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 10:49:25 +0100 Subject: [PATCH 03/79] Update README of multi-tenant-configuration script --- multi-tenant-configuration/README.md | 59 +++++++++ .../group_configuration_test.yaml | 113 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 multi-tenant-configuration/configurations/group_configuration_test.yaml diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 5b24ac0..533284b 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -1 +1,60 @@ # Multi-tenants User configuration scripts for Opencast +**ToDo** + +This script ... + +Currently this script does not ... + +## How to Use + +### Configuration +**ToDo** + +The script is configured by editing the values in `config.py`: + +| Configuration Key | Description | Default/Example | +| :---------------- | :---------------------------------------- | :--------------------------- | +| `url` | The URL of the global admin node ? | https://tenant1.opencast.com | +| `tenant_url_pattern` | The URL pattern of the target tenants | https://tenant2.opencast.com | +| `tenant_urls` | A dictioanry of server URLs of the target tenants | https://tenant2.opencast.com | +| `digest_user` | The user name of the digest user | opencast_system_account | +| `digest_pw` | The password of the digest user | CHANGE_ME | +| `env_path` | The id of the workflow to start on ingest | reimport-workflow | + +**TODo**: check the below ... + +_The configured digest user needs to exist on both tenants and have the same password for both of them. This is because +the script ingests the assets via URL, which is faster, but the user needs to be able to access the source tenant from +the target tenant for this to work. Additionally the user currently needs to have ROLE_ADMIN to be able to use +`/assets/{episodeid}`._ + +_For the future, Basic Authentication and the use of an endpoint that doesn't require the Admin role (e.g. +`api/events/{id}`) would be preferable, so you can simply add a frontend user with the necessary rights (ingest, +access to the events/series) and the same password to both tenants._ + +### Usage +**ToDo** + +The script can be called with the following parameters (all parameters in brackets are optional): + +`main.py ... ` + +| Short Option | Long Option | Description | +| :----------: | :---------- | :-------------------------------------------------------------- | +| `-t` | `--tenant` | The id(s) of the tenant to be configured | +| `-e` | `--environment` | The environment where to find the configuration file (either `staging` or `production`) | +| ... | ... | ... | + +#### Usage example +**ToDo** + +`main.py ... ` + +## Requirements +**ToDo** + +This scrypt was written for Python 3.8. You can install the necessary packages with + +`pip install -r requirements.txt` + +Additionally, this script uses modules contained in the _lib_ directory. diff --git a/multi-tenant-configuration/configurations/group_configuration_test.yaml b/multi-tenant-configuration/configurations/group_configuration_test.yaml new file mode 100644 index 0000000..a27ceb1 --- /dev/null +++ b/multi-tenant-configuration/configurations/group_configuration_test.yaml @@ -0,0 +1,113 @@ +{ + "groups" : [ + { + "name": "System Administrators", + "description": "System Administrators", + "tenants": "all", + "type": "closed", + "members": [ + { + "name": "Guy 1", + "email": "test@test.de", + "reason": "Operations partner", + "uid": "guy-1", + "tenants": "all" + }, + { + "name": "Guy 2", + "email": "test@test.de", + "reason": "Operations partner", + "uid": "guy-2", + "tenants": "tenant1" + } + ], + "inactive_members": [ ], + "permissions": [ + { + "tenants": "all", + "roles": ["ROLE_ADMIN", "ROLE_SUDO"] + } + ] + }, + { + "name": "Organization Administrators", + "description": "Organization administrators have full access to all content of ${name}", + "tenants": "all", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI", + "ROLE_ORG_ADMIN" + ] + }, + { + "tenants" : "tenant2", + "roles": { + "add": [ + "ROLE_UI_EVENTS_DETAILS_ACL_VIEW", + "ROLE_UI_EVENTS_DETAILS_ACL_EDIT" + ], + "remove": [] + } + } + ] + }, + { + "name": "Producers", + "description": "Producers have limited access to content and functionality", + "tenants": "all", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI", + "ROLE_UI_EVENTS_CREATE" + ] + }, + { + "tenants" : "tenant1", + "roles": { + "add": [ + "ROLE_UI_EVENTS_COUNTERS_VIEW" + ], + "remove": [ + "ROLE_UI_EVENTS_CREATE" + ] + } + }, + { + "tenants" : "tenant2", + "roles": { + "add": [ + "ROLE_ORG_ADMIN" + ], + "remove": [] + } + } + ] + }, + { + "name": "Tenant2 Producers", + "description": "Tenant2 Producers have limited access to content and functionality", + "tenants": "tenant2", + "type": "open", + "members": [], + "inactive_members": [], + "permissions": [ + { + "tenants": "all", + "roles": [ + "ROLE_ADMIN_UI" + ] + } + ] + } + ] +} From f40ad0d1e5239084f209e6947095005b5e0b4e7c Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 12:26:43 +0100 Subject: [PATCH 04/79] Added basic arguments parser for command options --- multi-tenant-configuration/main.py | 49 +++++++++++++---------------- multi-tenant-configuration/utils.py | 32 +++++++++++++++++++ 2 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 multi-tenant-configuration/utils.py diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 75e3866..bf39dab 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -1,51 +1,46 @@ import os import sys -import yaml - sys.path.append(os.path.join(os.path.abspath('..'), "lib")) -# import datetime -import config -import io -# from collections import defaultdict -from rest_requests.request_error import RequestError +# import io +import yaml +# from args.args_parser import get_args_parser +# from args.args_error import args_error +# from rest_requests.request_error import RequestError from args.digest_login import DigestLogin from rest_requests.request import get_request, post_request -# from pathlib import Path +from utils import parse_args, read_configuration_file +import config # ToDo # add logger # add interaction question -# add parameter to python command def main(): """ configure Groups and Users """ + environment, tenant_id = parse_args() + + print('environment: ', environment[0]) + print('tenant id: ', tenant_id) + + # create Digest Login digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # read config file opencast_organizations = read_configuration_file(config.env_path)['opencast_organizations'] - tenants = [tenant['id'] for tenant in opencast_organizations] - external_api_accounts = opencast_organizations[1]['external_api_accounts'] - - # create users for tenant 1 - for account in external_api_accounts: - tenant = tenants[1] - create_user(tenant, account, digest_login) - - -def read_configuration_file(path): - with open(path, 'r') as f: - conf = yaml.load(f, Loader=yaml.FullLoader) - - return conf + # tenants = [tenant['id'] for tenant in opencast_organizations] + # external_api_accounts = opencast_organizations[1]['external_api_accounts'] + # + # # create users for tenant 1 + # for account in external_api_accounts: + # tenant = tenants[1] + # response = create_user(tenant, account, digest_login) + # # json_content = get_json_content(response) + # # print(response) -# # example get request -# response = get_request("http://tenant1:8080/users/users.json", digest_login, "users/users.json") -# json_content = get_json_content(response) -# print(response) def get_roles_as_Json_array(account): roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py new file mode 100644 index 0000000..9fb87d7 --- /dev/null +++ b/multi-tenant-configuration/utils.py @@ -0,0 +1,32 @@ +import yaml + +from args.args_parser import get_args_parser +from args.args_error import args_error + + +def parse_args(): + """ + Parse the arguments and check them for correctness + + :return: list of event ids, list of series ids (one of them will be None) + :rtype: list, list + """ + parser, optional_args, required_args = get_args_parser() + + # change to required_args ? + required_args.add_argument("-e", "--environment", type=str, nargs='+', help="the environment (either 'staging' or 'production')") + optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") + + args = parser.parse_args() + + if not args.environment: + args_error(parser, "You have to provide an environment") + + return args.environment, args.tenantid + + +def read_configuration_file(path): + with open(path, 'r') as f: + conf = yaml.load(f, Loader=yaml.FullLoader) + + return conf \ No newline at end of file From 74bc708c3b7f5e4cd7c7a06be338d0d22c4d39fd Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 12:27:56 +0100 Subject: [PATCH 05/79] updated config file - added dictionary for tenant_urls - updated documentation --- multi-tenant-configuration/config.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 9307f0e..bd27f66 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -1,16 +1,19 @@ # Configuration -#Set this to your global admin node +# Set this to your global admin node url = "http://tenant1:8080" -#If you have multiple tenants use something like -#url_pattern = "https://{}.example.org" -#otherwise, url_pattern should be the same as the url variable above -url_pattern = "http://{}:8080" -# list of tenant URLs -tenant_urls = [ - 'http://tenant1:8080', - 'http://tenant2:8080' -] + +# If you have multiple tenants use an URL pattern: +# tenant_url_pattern = "https://{}.example.org" +# ToDo otherwise, this can be empty or commented out +tenant_url_pattern = "http://{}:8080" +# ToDo You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: +# example: +# tenant_urls = { '': 'http://tenant1:8080', '': 'http://tenant2:8080' } +tenant_urls = { + 'tenant1': 'http://tenant1:8080', + 'tenant2': 'http://tenant2:8080' +} # digest user digest_user = "opencast_system_account" From cd8fc84d6d7cce98f5c70e5afa9e6658a0ed6719 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 12:32:05 +0100 Subject: [PATCH 06/79] re-inserted API request for testing --- multi-tenant-configuration/main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index bf39dab..a1c1b10 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -31,15 +31,16 @@ def main(): digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # read config file opencast_organizations = read_configuration_file(config.env_path)['opencast_organizations'] - # tenants = [tenant['id'] for tenant in opencast_organizations] - # external_api_accounts = opencast_organizations[1]['external_api_accounts'] - # - # # create users for tenant 1 - # for account in external_api_accounts: - # tenant = tenants[1] - # response = create_user(tenant, account, digest_login) - # # json_content = get_json_content(response) - # # print(response) + + tenants = [tenant['id'] for tenant in opencast_organizations] + external_api_accounts = opencast_organizations[1]['external_api_accounts'] + + # create users for tenant 1 + for account in external_api_accounts: + tenant = tenants[1] + response = create_user(tenant, account, digest_login) + # json_content = get_json_content(response) + # print(response) def get_roles_as_Json_array(account): @@ -55,7 +56,7 @@ def create_user(tenantid, account, digest_login): :param digest_login: digest login :return: """ - url = '{}/admin-ng/users/'.format(config.url_pattern.format(tenantid)) + url = '{}/admin-ng/users/'.format(config.tenant_url_pattern.format(tenantid)) data = { 'username': account['username'], 'password': account['password'], From 97172191715cde095b778c18cbd9cd3c1c239aab Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Feb 2021 12:37:12 +0100 Subject: [PATCH 07/79] added newline --- multi-tenant-configuration/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py index 9fb87d7..0cc45c3 100644 --- a/multi-tenant-configuration/utils.py +++ b/multi-tenant-configuration/utils.py @@ -29,4 +29,4 @@ def read_configuration_file(path): with open(path, 'r') as f: conf = yaml.load(f, Loader=yaml.FullLoader) - return conf \ No newline at end of file + return conf From 4617bc6663c8a239a5b54e4363331a760fe6b8ab Mon Sep 17 00:00:00 2001 From: mheyen Date: Sat, 27 Feb 2021 12:40:47 +0100 Subject: [PATCH 08/79] minor restructuring of the script and improved options in available configuration most functions are now in utils.py . can be shifted to other libs if they are ready. tenant_urls can now be configured manually or are automatically generated by the script based on the tenant_url_pattern. --- multi-tenant-configuration/config.py | 10 ++-- multi-tenant-configuration/main.py | 62 +++++++++---------------- multi-tenant-configuration/utils.py | 69 ++++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 54 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index bd27f66..d71b512 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -10,14 +10,14 @@ # ToDo You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: # example: # tenant_urls = { '': 'http://tenant1:8080', '': 'http://tenant2:8080' } -tenant_urls = { - 'tenant1': 'http://tenant1:8080', - 'tenant2': 'http://tenant2:8080' -} +# tenant_urls = { +# 'tenant1': 'http://tenant1:8080', +# 'tenant2': 'http://tenant2:8080' +# } # digest user digest_user = "opencast_system_account" digest_pw = "CHANGE_ME" # path to environment configuration file -env_path = "environment/staging/opencast-organizations.yml" +env_path = "environment/{}/opencast-organizations.yml" diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index a1c1b10..d6b1c70 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -3,13 +3,13 @@ sys.path.append(os.path.join(os.path.abspath('..'), "lib")) # import io -import yaml +# import yaml # from args.args_parser import get_args_parser # from args.args_error import args_error # from rest_requests.request_error import RequestError +from input_output.input import get_yes_no_answer from args.digest_login import DigestLogin -from rest_requests.request import get_request, post_request -from utils import parse_args, read_configuration_file +from utils import parse_args, read_yaml_file, parse_config, create_user import config @@ -22,52 +22,32 @@ def main(): configure Groups and Users """ + # parse args environment, tenant_id = parse_args() - - print('environment: ', environment[0]) - print('tenant id: ', tenant_id) - + # read environment config file + env_conf = read_yaml_file(config.env_path.format(environment)) + # parse config.py + parse_config(config, env_conf) # create Digest Login digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) - # read config file - opencast_organizations = read_configuration_file(config.env_path)['opencast_organizations'] - tenants = [tenant['id'] for tenant in opencast_organizations] - external_api_accounts = opencast_organizations[1]['external_api_accounts'] + start_process = get_yes_no_answer("Create User?") + if not start_process: + __abort_script("Okay, not doing anything.") - # create users for tenant 1 + external_api_accounts = env_conf['opencast_organizations'][1]['external_api_accounts'] + # create user accounts on the specified tenant for account in external_api_accounts: - tenant = tenants[1] - response = create_user(tenant, account, digest_login) - # json_content = get_json_content(response) - # print(response) - - -def get_roles_as_Json_array(account): - roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + url = config.tenant_urls[tenant_id] + print(url) + response = create_user(account, digest_login, url) + json_content = get_json_content(response) + print(response) - return roles - -def create_user(tenantid, account, digest_login): - """ sends a POST request to the admin UI to create a User - - :param tenantid: str tenant id to form correct url (e.g. 'tenant1') - :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} - :param digest_login: digest login - :return: - """ - url = '{}/admin-ng/users/'.format(config.tenant_url_pattern.format(tenantid)) - data = { - 'username': account['username'], - 'password': account['password'], - 'name': account['name'], - 'email': account['email'], - 'roles': str(get_roles_as_Json_array(account)) - } - # ToDo error handling - response = post_request(url, digest_login, '/admin-ng/users/', data=data) - return response +def __abort_script(message): + print(message) + sys.exit() if __name__ == '__main__': diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py index 0cc45c3..1e2f967 100644 --- a/multi-tenant-configuration/utils.py +++ b/multi-tenant-configuration/utils.py @@ -1,15 +1,15 @@ import yaml - from args.args_parser import get_args_parser from args.args_error import args_error +from rest_requests.request import get_request, post_request def parse_args(): """ Parse the arguments and check them for correctness - :return: list of event ids, list of series ids (one of them will be None) - :rtype: list, list + :return: + :rtype: """ parser, optional_args, required_args = get_args_parser() @@ -20,13 +20,66 @@ def parse_args(): args = parser.parse_args() if not args.environment: - args_error(parser, "You have to provide an environment") + args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") + if not args.environment[0] in ('staging', 'production'): + args_error(parser, "The environment has to be either 'staging' or 'production'") + if len(args.environment) > 1: + args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") + + if not args.tenantid: + args.tenantid = [''] - return args.environment, args.tenantid + return args.environment[0], args.tenantid[0] -def read_configuration_file(path): +def read_yaml_file(path): + """ + reads a .yaml file and returns a dictionary + :param path: path to the yaml file + :return: returns a dictionary + """ + # ToDo error handling if path or file does not exist + # FileNotFoundError: with open(path, 'r') as f: - conf = yaml.load(f, Loader=yaml.FullLoader) + content = yaml.load(f, Loader=yaml.FullLoader) + + return content + + +def parse_config(config, env_config): + + tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations']] + if not (hasattr(config,'tenant_urls') and config.tenant_urls): + config.tenant_urls = {} + for tenant_id in tenant_ids: + config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) + + return True + + +def get_roles_as_Json_array(account): + roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + + return roles + +def create_user(account, digest_login, base_url): + """ sends a POST request to the admin UI to create a User + + :param tenantid: str tenant id to form correct url (e.g. 'tenant1') + :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} + :param digest_login: digest login + :param base_url: base url + :return: + """ + url = '{}/admin-ng/users/'.format(base_url) + data = { + 'username': account['username'], + 'password': account['password'], + 'name': account['name'], + 'email': account['email'], + 'roles': str(get_roles_as_Json_array(account)) + } + # ToDo error handling + response = post_request(url, digest_login, '/admin-ng/users/', data=data) - return conf + return response From 597260b8df34e757bfeaf45faf9817f0e06e164e Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Mar 2021 15:05:01 +0100 Subject: [PATCH 09/79] Added error handling for user creation and option to add user to all tenants --- multi-tenant-configuration/main.py | 23 ++++++++++++++++------- multi-tenant-configuration/utils.py | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index d6b1c70..e6e8909 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -36,14 +36,23 @@ def main(): __abort_script("Okay, not doing anything.") external_api_accounts = env_conf['opencast_organizations'][1]['external_api_accounts'] - # create user accounts on the specified tenant - for account in external_api_accounts: - url = config.tenant_urls[tenant_id] - print(url) - response = create_user(account, digest_login, url) - json_content = get_json_content(response) - print(response) + if not tenant_id: + for_all_tenants = get_yes_no_answer("Create User for all tenants?") + if for_all_tenants: + # create user account on all tenants + for tenant_url in config.tenant_urls: + for account in external_api_accounts: + response = create_user(account, digest_login, tenant_url) + # json_content = get_json_content(response) + print(response) + else: + # create user accounts on the specified tenant + for account in external_api_accounts: + tenant_url = config.tenant_urls[tenant_id] + response = create_user(account, digest_login, tenant_url) + if response: + print("created user {}".format(account['username'])) def __abort_script(message): print(message) diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py index 1e2f967..fea2a9d 100644 --- a/multi-tenant-configuration/utils.py +++ b/multi-tenant-configuration/utils.py @@ -2,6 +2,7 @@ from args.args_parser import get_args_parser from args.args_error import args_error from rest_requests.request import get_request, post_request +from rest_requests.request_error import RequestError def parse_args(): @@ -79,7 +80,16 @@ def create_user(account, digest_login, base_url): 'email': account['email'], 'roles': str(get_roles_as_Json_array(account)) } - # ToDo error handling - response = post_request(url, digest_login, '/admin-ng/users/', data=data) + try: + response = post_request(url, digest_login, '/admin-ng/users/', data=data) + except RequestError as err: + if err.get_status_code() == "409": + print("Conflict, a user with username {} already exist.".format(account['username'])) + if err.get_status_code() == "403": + print("Forbidden, not enough permissions to create a user with a admin role.") + return False + except Exception as e: + print("User could not be created: {}".format(str(e))) + return False return response From f26f524c97d5ad87e4b03ba6f0b666392af52e93 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Mar 2021 15:06:35 +0100 Subject: [PATCH 10/79] Update REAMDE --- multi-tenant-configuration/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 533284b..b2088d4 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -17,8 +17,8 @@ The script is configured by editing the values in `config.py`: | `url` | The URL of the global admin node ? | https://tenant1.opencast.com | | `tenant_url_pattern` | The URL pattern of the target tenants | https://tenant2.opencast.com | | `tenant_urls` | A dictioanry of server URLs of the target tenants | https://tenant2.opencast.com | -| `digest_user` | The user name of the digest user | opencast_system_account | -| `digest_pw` | The password of the digest user | CHANGE_ME | +| `digest_user` | The user name of the digest user | `opencast_system_account` | +| `digest_pw` | The password of the digest user | `CHANGE_ME` | | `env_path` | The id of the workflow to start on ingest | reimport-workflow | **TODo**: check the below ... @@ -39,11 +39,11 @@ The script can be called with the following parameters (all parameters in bracke `main.py ... ` -| Short Option | Long Option | Description | -| :----------: | :---------- | :-------------------------------------------------------------- | -| `-t` | `--tenant` | The id(s) of the tenant to be configured | -| `-e` | `--environment` | The environment where to find the configuration file (either `staging` or `production`) | -| ... | ... | ... | +| Param | Description | +| :---: | :---------- | +| `-t` / `--tenant` | The id(s) of the tenant to be configured | +| `-e` / `--environment` | The environment where to find the configuration file (either `staging` or `production`) | +| ... / ... | ... | #### Usage example **ToDo** From 2a23c7882aafcf1b5dd0a33bf1d6f6d4430901d0 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 25 Mar 2021 16:10:06 +0100 Subject: [PATCH 11/79] fixed minor error while creating users for all tenants --- multi-tenant-configuration/main.py | 28 +++++++++++++++------------- multi-tenant-configuration/utils.py | 6 ++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index e6e8909..6f01eaf 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -35,24 +35,26 @@ def main(): if not start_process: __abort_script("Okay, not doing anything.") - external_api_accounts = env_conf['opencast_organizations'][1]['external_api_accounts'] + # external_api_accounts = env_conf['opencast_organizations'][1]['external_api_accounts'] + external_api_accounts = {} + for tenant in env_conf['opencast_organizations']: + id = tenant['id'] + if id != "dummy": + external_api_accounts[id] = tenant['external_api_accounts'] if not tenant_id: for_all_tenants = get_yes_no_answer("Create User for all tenants?") - if for_all_tenants: - # create user account on all tenants - for tenant_url in config.tenant_urls: - for account in external_api_accounts: - response = create_user(account, digest_login, tenant_url) - # json_content = get_json_content(response) - print(response) + if not for_all_tenants: + __abort_script("Okay, not doing anything.") + else: + # create user account for all tenants + for tenant_id in config.tenant_ids: + for account in external_api_accounts[tenant_id]: + response = create_user(account, digest_login, config.tenant_urls[tenant_id]) else: # create user accounts on the specified tenant - for account in external_api_accounts: - tenant_url = config.tenant_urls[tenant_id] - response = create_user(account, digest_login, tenant_url) - if response: - print("created user {}".format(account['username'])) + for account in external_api_accounts[tenant_id]: + response = create_user(account, digest_login, config.tenant_urls[tenant_id]) def __abort_script(message): print(message) diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py index fea2a9d..faf018f 100644 --- a/multi-tenant-configuration/utils.py +++ b/multi-tenant-configuration/utils.py @@ -49,10 +49,11 @@ def read_yaml_file(path): def parse_config(config, env_config): - tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations']] + # ToDo check if "dummy" is realy how it should be in the organizations file + config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] if not (hasattr(config,'tenant_urls') and config.tenant_urls): config.tenant_urls = {} - for tenant_id in tenant_ids: + for tenant_id in config.tenant_ids: config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) return True @@ -82,6 +83,7 @@ def create_user(account, digest_login, base_url): } try: response = post_request(url, digest_login, '/admin-ng/users/', data=data) + print("created user {}".format(account['username'])) except RequestError as err: if err.get_status_code() == "409": print("Conflict, a user with username {} already exist.".format(account['username'])) From 417986f54d8c2649adb85913d1c06796c967f169 Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 29 Apr 2021 22:20:47 +0200 Subject: [PATCH 12/79] included group checks and restructured code --- multi-tenant-configuration/README.md | 4 +- multi-tenant-configuration/config.py | 4 +- .../configurations/group_configuration.yaml | 83 ++++++ .../group_configuration_test.yaml | 113 -------- .../configure_groups.py | 247 ++++++++++++++++++ multi-tenant-configuration/configure_users.py | 170 ++++++++++++ .../staging/opencast-organizations.yml | 4 +- multi-tenant-configuration/main.py | 201 ++++++++++---- .../parsing_configurations.py | 150 +++++++++++ multi-tenant-configuration/utils.py | 97 ------- 10 files changed, 813 insertions(+), 260 deletions(-) create mode 100644 multi-tenant-configuration/configurations/group_configuration.yaml delete mode 100644 multi-tenant-configuration/configurations/group_configuration_test.yaml create mode 100644 multi-tenant-configuration/configure_groups.py create mode 100644 multi-tenant-configuration/configure_users.py create mode 100644 multi-tenant-configuration/parsing_configurations.py delete mode 100644 multi-tenant-configuration/utils.py diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index b2088d4..d9543a8 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -8,7 +8,6 @@ Currently this script does not ... ## How to Use ### Configuration -**ToDo** The script is configured by editing the values in `config.py`: @@ -32,6 +31,9 @@ _For the future, Basic Authentication and the use of an endpoint that doesn't re `api/events/{id}`) would be preferable, so you can simply add a frontend user with the necessary rights (ingest, access to the events/series) and the same password to both tenants._ +#### group config: +The names in the group config file must be unique per Tenant! + ### Usage **ToDo** diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index d71b512..14c449d 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -8,7 +8,7 @@ # ToDo otherwise, this can be empty or commented out tenant_url_pattern = "http://{}:8080" # ToDo You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: -# example: +# # example: # tenant_urls = { '': 'http://tenant1:8080', '': 'http://tenant2:8080' } # tenant_urls = { # 'tenant1': 'http://tenant1:8080', @@ -21,3 +21,5 @@ # path to environment configuration file env_path = "environment/{}/opencast-organizations.yml" +# path to group configuration file +group_path = "configurations/group_configuration.yaml" diff --git a/multi-tenant-configuration/configurations/group_configuration.yaml b/multi-tenant-configuration/configurations/group_configuration.yaml new file mode 100644 index 0000000..52cd805 --- /dev/null +++ b/multi-tenant-configuration/configurations/group_configuration.yaml @@ -0,0 +1,83 @@ +--- + +groups: + - name: System Administrators + description: System Administrators + tenants: all + type: closed + members: + - name: Guy 1 + email: test@test.de + reason: Operations partner + uid: guy1 + tenants: all + - name: Guy 2 + email: test@test.de + reason: Operations partner + uid: guy2 + tenants: tenant1 + inactive_members: [] + permissions: + - tenants: all + roles: + - ROLE_ADMIN + - ROLE_SUDO + - name: Organization Administrators + description: Organization administrators have full access to all content of ${name} + tenants: all + type: open + members: [] + inactive_members: [] + permissions: + - tenants: all + roles: + - ROLE_ADMIN_UI + - ROLE_ORG_ADMIN + - tenants: tenant2 + roles: + add: + - ROLE_UI_EVENTS_DETAILS_ACL_VIEW + - ROLE_UI_EVENTS_DETAILS_ACL_EDIT + remove: [] + - name: Producers + description: Producers have limited access to content and functionality + tenants: all + type: open + members: [] + inactive_members: [] + permissions: + - tenants: all + roles: + - ROLE_ADMIN_UI + - ROLE_UI_EVENTS_CREATE + - tenants: tenant1 + roles: + add: + - ROLE_UI_EVENTS_COUNTERS_VIEW + remove: + - ROLE_UI_EVENTS_CREATE + - tenants: tenant2 + roles: + add: + - ROLE_ORG_ADMIN + remove: [] + - name: Tenant1 Producers + description: Tenant1 Producers have limited access to content and functionality + tenants: tenant1 + type: open + members: + - name: Guy X + email: test@test.de + reason: Operations partner + uid: guyx + tenants: all + - name: Guy 2 + email: test@test.de + reason: Operations partner + uid: guy2 + tenants: tenant1 + inactive_members: [] + permissions: + - tenants: all + roles: + - ROLE_ADMIN_UI diff --git a/multi-tenant-configuration/configurations/group_configuration_test.yaml b/multi-tenant-configuration/configurations/group_configuration_test.yaml deleted file mode 100644 index a27ceb1..0000000 --- a/multi-tenant-configuration/configurations/group_configuration_test.yaml +++ /dev/null @@ -1,113 +0,0 @@ -{ - "groups" : [ - { - "name": "System Administrators", - "description": "System Administrators", - "tenants": "all", - "type": "closed", - "members": [ - { - "name": "Guy 1", - "email": "test@test.de", - "reason": "Operations partner", - "uid": "guy-1", - "tenants": "all" - }, - { - "name": "Guy 2", - "email": "test@test.de", - "reason": "Operations partner", - "uid": "guy-2", - "tenants": "tenant1" - } - ], - "inactive_members": [ ], - "permissions": [ - { - "tenants": "all", - "roles": ["ROLE_ADMIN", "ROLE_SUDO"] - } - ] - }, - { - "name": "Organization Administrators", - "description": "Organization administrators have full access to all content of ${name}", - "tenants": "all", - "type": "open", - "members": [], - "inactive_members": [], - "permissions": [ - { - "tenants": "all", - "roles": [ - "ROLE_ADMIN_UI", - "ROLE_ORG_ADMIN" - ] - }, - { - "tenants" : "tenant2", - "roles": { - "add": [ - "ROLE_UI_EVENTS_DETAILS_ACL_VIEW", - "ROLE_UI_EVENTS_DETAILS_ACL_EDIT" - ], - "remove": [] - } - } - ] - }, - { - "name": "Producers", - "description": "Producers have limited access to content and functionality", - "tenants": "all", - "type": "open", - "members": [], - "inactive_members": [], - "permissions": [ - { - "tenants": "all", - "roles": [ - "ROLE_ADMIN_UI", - "ROLE_UI_EVENTS_CREATE" - ] - }, - { - "tenants" : "tenant1", - "roles": { - "add": [ - "ROLE_UI_EVENTS_COUNTERS_VIEW" - ], - "remove": [ - "ROLE_UI_EVENTS_CREATE" - ] - } - }, - { - "tenants" : "tenant2", - "roles": { - "add": [ - "ROLE_ORG_ADMIN" - ], - "remove": [] - } - } - ] - }, - { - "name": "Tenant2 Producers", - "description": "Tenant2 Producers have limited access to content and functionality", - "tenants": "tenant2", - "type": "open", - "members": [], - "inactive_members": [], - "permissions": [ - { - "tenants": "all", - "roles": [ - "ROLE_ADMIN_UI" - ] - } - ] - } - ] -} diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py new file mode 100644 index 0000000..9acaaed --- /dev/null +++ b/multi-tenant-configuration/configure_groups.py @@ -0,0 +1,247 @@ +import yaml +import json +from args.args_parser import get_args_parser +from args.args_error import args_error +from rest_requests.request import get_request, post_request +from rest_requests.request_error import RequestError +from input_output.input import get_yes_no_answer + + +# def parse_args(): +# """ +# Parse the arguments and check them for correctness +# +# :return: +# :rtype: +# """ +# parser, optional_args, required_args = get_args_parser() +# +# # ToDo change optional to required_args ? +# required_args.add_argument("-e", "--environment", type=str, nargs='+', +# help="the environment (either 'staging' or 'production')") +# optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") +# optional_args.add_argument("-c", "--check", type=str, nargs='+', +# help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") +# +# args = parser.parse_args() +# +# if not args.environment: +# args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") +# if not args.environment[0] in ('staging', 'production'): +# args_error(parser, "The environment has to be either 'staging' or 'production'") +# if len(args.environment) > 1: +# args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") +# +# if not args.tenantid: +# args.tenantid = [''] +# if not args.check: +# args.check = ['all'] +# +# return args.environment[0], args.tenantid[0], args.check[0] +# +# +# def read_yaml_file(path): +# """ +# reads a .yaml file and returns a dictionary +# :param path: path to the yaml file +# :return: returns a dictionary +# """ +# # ToDo error handling if path or file does not exist +# # FileNotFoundError: +# with open(path, 'r') as f: +# content = yaml.load(f, Loader=yaml.FullLoader) +# +# return content +# +# +# def parse_config(config, env_config): +# +# # ToDo check if "dummy" is really how it should be in the organizations file +# config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] +# # ToDo suche get all tenant funktion +# if not (hasattr(config,'tenant_urls') and config.tenant_urls): +# config.tenant_urls = {} +# for tenant_id in config.tenant_ids: +# config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) +# +# return True # ToDo return config ? +# +# +# def get_roles_as_Json_array(account): +# roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] +# +# return roles + +def check_groups(tenant_id, digest_login, group_config, config): + + tenant_url = config.tenant_urls[tenant_id] + # For all Groups: + for group in group_config['groups']: + if not tenant_id: + tenant_id = group['tenants'] + group['identifier'] = generate_group_identifier(group, tenant_id) + # Check group + if group['tenants'] == 'all' or group['tenants'] == tenant_id: + check_group(tenant_url=tenant_url, digest_login=digest_login, group=group, tenant_id=tenant_id) + + +def check_if_group_exists(tenant_url, digest_login, group, tenant_id): + # ToDo log + url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) + try: + response = get_request(url, digest_login, '/api/groups/') + return response.json() + except RequestError as err: + if err.get_status_code() == "404": + print('Group was not found: ', err) + return False + else: + raise Exception + except Exception as e: + print("ERROR: {}".format(str(e))) + return False + + +def check_group(tenant_url, digest_login, group, tenant_id): + # ToDo log + print(f"Checking group {group['name']} with id {group['identifier']}") + + # Check if group exists. + existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) + if not existing_group: + # Create group if it does not exist. Ask for permission + answer = get_yes_no_answer(f"group {group['name']} does not exist. Create group?") + if answer: + existing_group = create_group(group=group, digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id) + elif existing_group: + # Check if group name and description match the name and description provided in the configuration. + print('check names:') + if group['name'] == existing_group['name']: + print('names are equal') + # Update them if they do not match. (Asks for permission) + # Check if group members exist. + # Create missing group members. (Asks for permission) + # Check if group roles match the group roles provided in the configuration. + # Update group roles if they do not match.(Asks for permission) + # Check if group members match the group members provided in the configuration. Add or remove members accordingly. + # Check external API accounts of members. Add missing API accounts. + # Check group type. If group is closed, remove unexpected members. + # Update group members. (Asks for permission) + + +def generate_group_identifier(group, tenant_id): + # ToDo move this to parse group config file + # ToDo check if the generated identifiers are correct! (the same as in the ruby script) + # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() + return group['name'].replace(' ', '_').lower() + + +# def create_user(account, digest_login, tenant_url): +# """ sends a POST request to the admin UI to create a User +# +# :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} +# :param digest_login: digest login +# :param tenant_url: tenant url +# :return: +# """ +# url = '{}/admin-ng/users/'.format(tenant_url) +# data = { +# 'username': account['username'], +# 'password': account['password'], +# 'name': account['name'], +# 'email': account['email'], +# 'roles': str(get_roles_as_Json_array(account)) +# } +# try: +# response = post_request(url, digest_login, '/admin-ng/users/', data=data) +# print("created user {}".format(account['username'])) +# except RequestError as err: +# if err.get_status_code() == "409": +# print("Conflict, a user with username {} already exist.".format(account['username'])) +# if err.get_status_code() == "403": +# print("Forbidden, not enough permissions to create a user with a admin role.") +# return False +# except Exception as e: +# print("User could not be created: {}".format(str(e))) +# return False +# +# return response +# + +def get_groups_from_tenant(tenant_url, digest_login): + + url = '{}/api/groups/'.format(tenant_url) + try: + response = get_request(url, digest_login, '/api/groups/') + except RequestError as err: + print('RequestError:') + print(err) + return False + except Exception as e: + print("Groups could not be retrieved from {}. ".format(tenant_url)) + print("Error: {}".format(str(e))) + return False + + return response.json() + + +def create_group(group, digest_login, tenant_url, tenant_id): + # ToDo log + print('Try to create Group!') + url = '{}/api/groups/'.format(tenant_url) + + # ToDo is this logic correct? + # should be checked if the member exists? + members = [member['uid'] for member in group['members'] + if member['tenants'] == 'all' or member['tenants'] == tenant_id] + members = ",".join(members) + # print("members: ", members) + + # ToDo check group config file if 'add' and 'remove' are needed + roles = [] + for permission in group['permissions']: + if permission['tenants'] == 'all' or permission['tenants'] == tenant_id: + for role in permission['roles']: + roles.append(role) + roles = ','.join(roles) + # print("roles: ", roles) + + data = { + 'name': group['name'], + 'description': group['description'], + 'roles': roles, + 'members': members, + } + try: + response = post_request(url, digest_login, '/api/groups/', data=data) + print("created group {}".format(group['name'])) + except RequestError as err: + if err.get_status_code() == "400": + print("Conflict: group with name {} could not be created.".format(group['name'])) + elif err.get_status_code() == "409": + print("Failed to create group: ", err) + else: + print(err) + return False + except Exception as e: + print("Group could not be created: {}".format(str(e))) + return False + + return response + + +# def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): +# """ +# This function can be used to transform a json file to a yaml file. +# requires import json and import yaml +# :param json_file_path: path to json file +# :param yaml_file_path: path to yaml file (will be created if it does not exist) +# :return: +# """ +# +# with open(json_file_path, 'r') as json_file: +# jsonData = json.load(json_file) +# with open(yaml_file_path, 'w') as file: +# yaml.dump(jsonData, file, sort_keys=False) +# +# return True \ No newline at end of file diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py new file mode 100644 index 0000000..efad8ed --- /dev/null +++ b/multi-tenant-configuration/configure_users.py @@ -0,0 +1,170 @@ +import yaml +import json +from args.args_parser import get_args_parser +from args.args_error import args_error +from rest_requests.request import get_request, post_request +from rest_requests.request_error import RequestError +from input_output.input import get_yes_no_answer +from parsing_configurations import __abort_script + +# def parse_args(): +# """ +# Parse the arguments and check them for correctness +# +# :return: +# :rtype: +# """ +# parser, optional_args, required_args = get_args_parser() +# +# # ToDo change optional to required_args ? +# required_args.add_argument("-e", "--environment", type=str, nargs='+', +# help="the environment (either 'staging' or 'production')") +# optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") +# optional_args.add_argument("-c", "--check", type=str, nargs='+', +# help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") +# +# args = parser.parse_args() +# +# if not args.environment: +# args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") +# if not args.environment[0] in ('staging', 'production'): +# args_error(parser, "The environment has to be either 'staging' or 'production'") +# if len(args.environment) > 1: +# args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") +# +# if not args.tenantid: +# args.tenantid = [''] +# if not args.check: +# args.check = ['all'] +# +# return args.environment[0], args.tenantid[0], args.check[0] +# +# +# def read_yaml_file(path): +# """ +# reads a .yaml file and returns a dictionary +# :param path: path to the yaml file +# :return: returns a dictionary +# """ +# # ToDo error handling if path or file does not exist +# # FileNotFoundError: +# with open(path, 'r') as f: +# content = yaml.load(f, Loader=yaml.FullLoader) +# +# return content +# +# +# def parse_config(config, env_config): +# +# # ToDo check if "dummy" is really how it should be in the organizations file +# config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] +# # ToDo suche get all tenant funktion +# if not (hasattr(config,'tenant_urls') and config.tenant_urls): +# config.tenant_urls = {} +# for tenant_id in config.tenant_ids: +# config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) +# +# return True # ToDo return config ? + + +def check_users(tenant_id, digest_login, env_conf, config): + print('Log: start checking users for tenant ', tenant_id) + + external_api_accounts = {} + for tenant in env_conf['opencast_organizations']: + id = tenant['id'] + if id != "dummy": + external_api_accounts[id] = tenant['external_api_accounts'] + + if not tenant_id: + for_all_tenants = get_yes_no_answer("Create User for all tenants?") + if not for_all_tenants: + __abort_script("Okay, not doing anything.") + else: + # create user account for all tenants + for tenant_id in config.tenant_ids: + for account in external_api_accounts[tenant_id]: + response = create_user(account, digest_login, config.tenant_urls[tenant_id]) + else: + # create user accounts on the specified tenant + for account in external_api_accounts[tenant_id]: + response = create_user(account, digest_login, config.tenant_urls[tenant_id]) + + +def get_roles_as_Json_array(account): + roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + + return roles + +# +# def generate_group_identifier(group, tenant_id): +# # ToDo move this to parse group config file +# # ToDo check if the generated identifiers are correct! (the same as in the ruby script) +# # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() +# return group['name'].replace(' ', '_').lower() + + +def create_user(account, digest_login, tenant_url): + """ sends a POST request to the admin UI to create a User + + :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} + :param digest_login: digest login + :param tenant_url: tenant url + :return: + """ + url = '{}/admin-ng/users/'.format(tenant_url) + data = { + 'username': account['username'], + 'password': account['password'], + 'name': account['name'], + 'email': account['email'], + 'roles': str(get_roles_as_Json_array(account)) + } + try: + response = post_request(url, digest_login, '/admin-ng/users/', data=data) + print("created user {}".format(account['username'])) + except RequestError as err: + if err.get_status_code() == "409": + print("Conflict, a user with username {} already exist.".format(account['username'])) + if err.get_status_code() == "403": + print("Forbidden, not enough permissions to create a user with a admin role.") + return False + except Exception as e: + print("User could not be created: {}".format(str(e))) + return False + + return response + + +# def get_groups_from_tenant(tenant_url, digest_login): +# +# url = '{}/api/groups/'.format(tenant_url) +# try: +# response = get_request(url, digest_login, '/api/groups/') +# except RequestError as err: +# print('RequestError:') +# print(err) +# return False +# except Exception as e: +# print("Groups could not be retrieved from {}. ".format(tenant_url)) +# print("Error: {}".format(str(e))) +# return False +# +# return response.json() + + +# def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): +# """ +# This function can be used to transform a json file to a yaml file. +# requires import json and import yaml +# :param json_file_path: path to json file +# :param yaml_file_path: path to yaml file (will be created if it does not exist) +# :return: +# """ +# +# with open(json_file_path, 'r') as json_file: +# jsonData = json.load(json_file) +# with open(yaml_file_path, 'w') as file: +# yaml.dump(jsonData, file, sort_keys=False) +# +# return True \ No newline at end of file diff --git a/multi-tenant-configuration/environment/staging/opencast-organizations.yml b/multi-tenant-configuration/environment/staging/opencast-organizations.yml index 25320fd..0714693 100644 --- a/multi-tenant-configuration/environment/staging/opencast-organizations.yml +++ b/multi-tenant-configuration/environment/staging/opencast-organizations.yml @@ -67,8 +67,8 @@ opencast_organizations: name: Moodle System User email: test@test.de roles: [ROLE_EXTERNAL_APPLICATION] - - username: guy-1 + - username: guyx password: abc - name: Guy 1 + name: Guy X email: test@test.de roles: [ROLE_ADMIN, ROLE_SUDO] diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 6f01eaf..3f46e74 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -7,58 +7,167 @@ # from args.args_parser import get_args_parser # from args.args_error import args_error # from rest_requests.request_error import RequestError -from input_output.input import get_yes_no_answer +# from input_output.input import get_yes_no_answer from args.digest_login import DigestLogin -from utils import parse_args, read_yaml_file, parse_config, create_user +from parsing_configurations import parse_args, read_yaml_file, parse_config +from configure_users import check_users +from configure_groups import check_groups +# from rest_requests.request import get_request, post_request +# from rest_requests.request_error import RequestError import config -# ToDo -# add logger -# add interaction question +VERBOSE_FLAG = True def main(): - """ - configure Groups and Users - """ - - # parse args - environment, tenant_id = parse_args() - # read environment config file - env_conf = read_yaml_file(config.env_path.format(environment)) - # parse config.py - parse_config(config, env_conf) - # create Digest Login - digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) - - start_process = get_yes_no_answer("Create User?") - if not start_process: - __abort_script("Okay, not doing anything.") - - # external_api_accounts = env_conf['opencast_organizations'][1]['external_api_accounts'] - external_api_accounts = {} - for tenant in env_conf['opencast_organizations']: - id = tenant['id'] - if id != "dummy": - external_api_accounts[id] = tenant['external_api_accounts'] - - if not tenant_id: - for_all_tenants = get_yes_no_answer("Create User for all tenants?") - if not for_all_tenants: - __abort_script("Okay, not doing anything.") - else: - # create user account for all tenants - for tenant_id in config.tenant_ids: - for account in external_api_accounts[tenant_id]: - response = create_user(account, digest_login, config.tenant_urls[tenant_id]) - else: - # create user accounts on the specified tenant - for account in external_api_accounts[tenant_id]: - response = create_user(account, digest_login, config.tenant_urls[tenant_id]) - -def __abort_script(message): - print(message) - sys.exit() + + ### Parse args and config ### + environment, tenant_id, check = parse_args() # parse args + env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file + script_config = parse_config(config, env_conf) # parse config.py + group_config = read_yaml_file(config.group_path) # read group config file + # ToDo Think about whether we should exclude Digest Login credentials from config.py file + digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # create Digest Login + + ### Start checks ### + if check == 'all': + check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) + check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + elif check == 'users': + check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) + elif check == 'groups': + check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + + +# def check_users(tenant_id, digest_login, env_conf): +# print('Log: start checking users for tenant ', tenant_id) +# +# external_api_accounts = {} +# for tenant in env_conf['opencast_organizations']: +# id = tenant['id'] +# if id != "dummy": +# external_api_accounts[id] = tenant['external_api_accounts'] +# +# if not tenant_id: +# for_all_tenants = get_yes_no_answer("Create User for all tenants?") +# if not for_all_tenants: +# __abort_script("Okay, not doing anything.") +# else: +# # create user account for all tenants +# for tenant_id in config.tenant_ids: +# for account in external_api_accounts[tenant_id]: +# response = create_user(account, digest_login, config.tenant_urls[tenant_id]) +# else: +# # create user accounts on the specified tenant +# for account in external_api_accounts[tenant_id]: +# response = create_user(account, digest_login, config.tenant_urls[tenant_id]) + + +# def check_groups(tenant_id, digest_login, group_config): +# +# tenant_url = config.tenant_urls[tenant_id] +# # For all Groups: +# for group in group_config['groups']: +# if not tenant_id: +# tenant_id = group['tenants'] +# group['identifier'] = generate_group_identifier(group, tenant_id) +# # Check group +# if group['tenants'] == 'all' or group['tenants'] == tenant_id: +# check_group(tenant_url=tenant_url, digest_login=digest_login, group=group, tenant_id=tenant_id) + + +# def check_if_group_exists(tenant_url, digest_login, group, tenant_id): +# # ToDo log +# url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) +# try: +# response = get_request(url, digest_login, '/api/groups/') +# return response.json() +# except RequestError as err: +# if err.get_status_code() == "404": +# print('Group was not found: ', err) +# return False +# else: +# raise Exception +# except Exception as e: +# print("ERROR: {}".format(str(e))) +# return False + +# +# def check_group(tenant_url, digest_login, group, tenant_id): +# # ToDo log +# print(f"Checking group {group['name']} with id {group['identifier']}") +# +# # Check if group exists. +# existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) +# if not existing_group: +# # Create group if it does not exist. Ask for permission +# answer = get_yes_no_answer(f"group {group['name']} does not exist. Create group?") +# if answer: +# existing_group = create_group(group=group, digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id) +# elif existing_group: +# # Check if group name and description match the name and description provided in the configuration. +# print('check names:') +# if group['name'] == existing_group['name']: +# print('names are equal') +# # Update them if they do not match. (Asks for permission) +# # Check if group members exist. +# # Create missing group members. (Asks for permission) +# # Check if group roles match the group roles provided in the configuration. +# # Update group roles if they do not match.(Asks for permission) +# # Check if group members match the group members provided in the configuration. Add or remove members accordingly. +# # Check external API accounts of members. Add missing API accounts. +# # Check group type. If group is closed, remove unexpected members. +# # Update group members. (Asks for permission) + + +# def create_group(group, digest_login, tenant_url, tenant_id): +# # ToDo log +# print('Try to create Group!') +# url = '{}/api/groups/'.format(tenant_url) +# +# # ToDo is this logic correct? +# # should be checked if the member exists? +# members = [member['uid'] for member in group['members'] +# if member['tenants'] == 'all' or member['tenants'] == tenant_id] +# members = ",".join(members) +# # print("members: ", members) +# +# # ToDo check group config file if 'add' and 'remove' are needed +# roles = [] +# for permission in group['permissions']: +# if permission['tenants'] == 'all' or permission['tenants'] == tenant_id: +# for role in permission['roles']: +# roles.append(role) +# roles = ','.join(roles) +# # print("roles: ", roles) +# +# data = { +# 'name': group['name'], +# 'description': group['description'], +# 'roles': roles, +# 'members': members, +# } +# try: +# response = post_request(url, digest_login, '/api/groups/', data=data) +# print("created group {}".format(group['name'])) +# except RequestError as err: +# if err.get_status_code() == "400": +# print("Conflict: group with name {} could not be created.".format(group['name'])) +# elif err.get_status_code() == "409": +# print("Failed to create group: ", err) +# else: +# print(err) +# return False +# except Exception as e: +# print("Group could not be created: {}".format(str(e))) +# return False +# +# return response + + +def log(message): + if(VERBOSE_FLAG): + print(message) if __name__ == '__main__': diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py new file mode 100644 index 0000000..6a317dd --- /dev/null +++ b/multi-tenant-configuration/parsing_configurations.py @@ -0,0 +1,150 @@ +import yaml +import json +from args.args_parser import get_args_parser +from args.args_error import args_error +from rest_requests.request import get_request, post_request +from rest_requests.request_error import RequestError + + +def parse_args(): + """ + Parse the arguments and check them for correctness + + :return: + :rtype: + """ + parser, optional_args, required_args = get_args_parser() + + # ToDo change optional to required_args ? + required_args.add_argument("-e", "--environment", type=str, nargs='+', + help="the environment (either 'staging' or 'production')") + optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") + optional_args.add_argument("-c", "--check", type=str, nargs='+', + help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") + + args = parser.parse_args() + + if not args.environment: + args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") + if not args.environment[0] in ('staging', 'production'): + args_error(parser, "The environment has to be either 'staging' or 'production'") + if len(args.environment) > 1: + args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") + + if not args.tenantid: + args.tenantid = [''] + if not args.check: + args.check = ['all'] + + return args.environment[0], args.tenantid[0], args.check[0] + + +def read_yaml_file(path): + """ + reads a .yaml file and returns a dictionary + :param path: path to the yaml file + :return: returns a dictionary + """ + # ToDo error handling if path or file does not exist + # FileNotFoundError: + with open(path, 'r') as f: + content = yaml.load(f, Loader=yaml.FullLoader) + + return content + + +def parse_config(config, env_config): + + # ToDo check if "dummy" is really how it should be in the organizations file + config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] + # ToDo suche get all tenant funktion + if not (hasattr(config,'tenant_urls') and config.tenant_urls): + config.tenant_urls = {} + for tenant_id in config.tenant_ids: + config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) + + return config + + +# def get_roles_as_Json_array(account): +# roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] +# +# return roles + + +# def generate_group_identifier(group, tenant_id): +# # ToDo move this to parse group config file +# # ToDo check if the generated identifiers are correct! (the same as in the ruby script) +# # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() +# return group['name'].replace(' ', '_').lower() + +# +# def create_user(account, digest_login, tenant_url): +# """ sends a POST request to the admin UI to create a User +# +# :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} +# :param digest_login: digest login +# :param tenant_url: tenant url +# :return: +# """ +# url = '{}/admin-ng/users/'.format(tenant_url) +# data = { +# 'username': account['username'], +# 'password': account['password'], +# 'name': account['name'], +# 'email': account['email'], +# 'roles': str(get_roles_as_Json_array(account)) +# } +# try: +# response = post_request(url, digest_login, '/admin-ng/users/', data=data) +# print("created user {}".format(account['username'])) +# except RequestError as err: +# if err.get_status_code() == "409": +# print("Conflict, a user with username {} already exist.".format(account['username'])) +# if err.get_status_code() == "403": +# print("Forbidden, not enough permissions to create a user with a admin role.") +# return False +# except Exception as e: +# print("User could not be created: {}".format(str(e))) +# return False +# +# return response + + +# def get_groups_from_tenant(tenant_url, digest_login): +# +# url = '{}/api/groups/'.format(tenant_url) +# try: +# response = get_request(url, digest_login, '/api/groups/') +# except RequestError as err: +# print('RequestError:') +# print(err) +# return False +# except Exception as e: +# print("Groups could not be retrieved from {}. ".format(tenant_url)) +# print("Error: {}".format(str(e))) +# return False +# +# return response.json() + + +def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): + """ + This function can be used to transform a json file to a yaml file. + requires import json and import yaml + :param json_file_path: path to json file + :param yaml_file_path: path to yaml file (will be created if it does not exist) + :return: + """ + + with open(json_file_path, 'r') as json_file: + jsonData = json.load(json_file) + with open(yaml_file_path, 'w') as file: + yaml.dump(jsonData, file, sort_keys=False) + + return True + + +def __abort_script(message): + print(message) + sys.exit() diff --git a/multi-tenant-configuration/utils.py b/multi-tenant-configuration/utils.py deleted file mode 100644 index faf018f..0000000 --- a/multi-tenant-configuration/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -import yaml -from args.args_parser import get_args_parser -from args.args_error import args_error -from rest_requests.request import get_request, post_request -from rest_requests.request_error import RequestError - - -def parse_args(): - """ - Parse the arguments and check them for correctness - - :return: - :rtype: - """ - parser, optional_args, required_args = get_args_parser() - - # change to required_args ? - required_args.add_argument("-e", "--environment", type=str, nargs='+', help="the environment (either 'staging' or 'production')") - optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") - - args = parser.parse_args() - - if not args.environment: - args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") - if not args.environment[0] in ('staging', 'production'): - args_error(parser, "The environment has to be either 'staging' or 'production'") - if len(args.environment) > 1: - args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") - - if not args.tenantid: - args.tenantid = [''] - - return args.environment[0], args.tenantid[0] - - -def read_yaml_file(path): - """ - reads a .yaml file and returns a dictionary - :param path: path to the yaml file - :return: returns a dictionary - """ - # ToDo error handling if path or file does not exist - # FileNotFoundError: - with open(path, 'r') as f: - content = yaml.load(f, Loader=yaml.FullLoader) - - return content - - -def parse_config(config, env_config): - - # ToDo check if "dummy" is realy how it should be in the organizations file - config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] - if not (hasattr(config,'tenant_urls') and config.tenant_urls): - config.tenant_urls = {} - for tenant_id in config.tenant_ids: - config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) - - return True - - -def get_roles_as_Json_array(account): - roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] - - return roles - -def create_user(account, digest_login, base_url): - """ sends a POST request to the admin UI to create a User - - :param tenantid: str tenant id to form correct url (e.g. 'tenant1') - :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} - :param digest_login: digest login - :param base_url: base url - :return: - """ - url = '{}/admin-ng/users/'.format(base_url) - data = { - 'username': account['username'], - 'password': account['password'], - 'name': account['name'], - 'email': account['email'], - 'roles': str(get_roles_as_Json_array(account)) - } - try: - response = post_request(url, digest_login, '/admin-ng/users/', data=data) - print("created user {}".format(account['username'])) - except RequestError as err: - if err.get_status_code() == "409": - print("Conflict, a user with username {} already exist.".format(account['username'])) - if err.get_status_code() == "403": - print("Forbidden, not enough permissions to create a user with a admin role.") - return False - except Exception as e: - print("User could not be created: {}".format(str(e))) - return False - - return response From 947ee773e78d7366bd6d3b1ba3ec467f18a0f38a Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 29 Apr 2021 23:04:43 +0200 Subject: [PATCH 13/79] restructured code and included verbose flag and logging --- .../configure_groups.py | 123 +--------------- multi-tenant-configuration/configure_users.py | 104 +------------- multi-tenant-configuration/main.py | 133 ------------------ .../parsing_configurations.py | 81 ++--------- 4 files changed, 21 insertions(+), 420 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 9acaaed..5ec9970 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -5,72 +5,7 @@ from rest_requests.request import get_request, post_request from rest_requests.request_error import RequestError from input_output.input import get_yes_no_answer - - -# def parse_args(): -# """ -# Parse the arguments and check them for correctness -# -# :return: -# :rtype: -# """ -# parser, optional_args, required_args = get_args_parser() -# -# # ToDo change optional to required_args ? -# required_args.add_argument("-e", "--environment", type=str, nargs='+', -# help="the environment (either 'staging' or 'production')") -# optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") -# optional_args.add_argument("-c", "--check", type=str, nargs='+', -# help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") -# -# args = parser.parse_args() -# -# if not args.environment: -# args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") -# if not args.environment[0] in ('staging', 'production'): -# args_error(parser, "The environment has to be either 'staging' or 'production'") -# if len(args.environment) > 1: -# args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") -# -# if not args.tenantid: -# args.tenantid = [''] -# if not args.check: -# args.check = ['all'] -# -# return args.environment[0], args.tenantid[0], args.check[0] -# -# -# def read_yaml_file(path): -# """ -# reads a .yaml file and returns a dictionary -# :param path: path to the yaml file -# :return: returns a dictionary -# """ -# # ToDo error handling if path or file does not exist -# # FileNotFoundError: -# with open(path, 'r') as f: -# content = yaml.load(f, Loader=yaml.FullLoader) -# -# return content -# -# -# def parse_config(config, env_config): -# -# # ToDo check if "dummy" is really how it should be in the organizations file -# config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] -# # ToDo suche get all tenant funktion -# if not (hasattr(config,'tenant_urls') and config.tenant_urls): -# config.tenant_urls = {} -# for tenant_id in config.tenant_ids: -# config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) -# -# return True # ToDo return config ? -# -# -# def get_roles_as_Json_array(account): -# roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] -# -# return roles +from parsing_configurations import log def check_groups(tenant_id, digest_login, group_config, config): @@ -103,8 +38,7 @@ def check_if_group_exists(tenant_url, digest_login, group, tenant_id): def check_group(tenant_url, digest_login, group, tenant_id): - # ToDo log - print(f"Checking group {group['name']} with id {group['identifier']}") + log(f"Check group {group['name']} with id {group['identifier']}") # Check if group exists. existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) @@ -115,9 +49,9 @@ def check_group(tenant_url, digest_login, group, tenant_id): existing_group = create_group(group=group, digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id) elif existing_group: # Check if group name and description match the name and description provided in the configuration. - print('check names:') + log('check names for group') if group['name'] == existing_group['name']: - print('names are equal') + log('names are equal') # Update them if they do not match. (Asks for permission) # Check if group members exist. # Create missing group members. (Asks for permission) @@ -136,38 +70,6 @@ def generate_group_identifier(group, tenant_id): return group['name'].replace(' ', '_').lower() -# def create_user(account, digest_login, tenant_url): -# """ sends a POST request to the admin UI to create a User -# -# :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} -# :param digest_login: digest login -# :param tenant_url: tenant url -# :return: -# """ -# url = '{}/admin-ng/users/'.format(tenant_url) -# data = { -# 'username': account['username'], -# 'password': account['password'], -# 'name': account['name'], -# 'email': account['email'], -# 'roles': str(get_roles_as_Json_array(account)) -# } -# try: -# response = post_request(url, digest_login, '/admin-ng/users/', data=data) -# print("created user {}".format(account['username'])) -# except RequestError as err: -# if err.get_status_code() == "409": -# print("Conflict, a user with username {} already exist.".format(account['username'])) -# if err.get_status_code() == "403": -# print("Forbidden, not enough permissions to create a user with a admin role.") -# return False -# except Exception as e: -# print("User could not be created: {}".format(str(e))) -# return False -# -# return response -# - def get_groups_from_tenant(tenant_url, digest_login): url = '{}/api/groups/'.format(tenant_url) @@ -228,20 +130,3 @@ def create_group(group, digest_login, tenant_url, tenant_id): return False return response - - -# def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): -# """ -# This function can be used to transform a json file to a yaml file. -# requires import json and import yaml -# :param json_file_path: path to json file -# :param yaml_file_path: path to yaml file (will be created if it does not exist) -# :return: -# """ -# -# with open(json_file_path, 'r') as json_file: -# jsonData = json.load(json_file) -# with open(yaml_file_path, 'w') as file: -# yaml.dump(jsonData, file, sort_keys=False) -# -# return True \ No newline at end of file diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index efad8ed..a4697b4 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -5,70 +5,11 @@ from rest_requests.request import get_request, post_request from rest_requests.request_error import RequestError from input_output.input import get_yes_no_answer -from parsing_configurations import __abort_script - -# def parse_args(): -# """ -# Parse the arguments and check them for correctness -# -# :return: -# :rtype: -# """ -# parser, optional_args, required_args = get_args_parser() -# -# # ToDo change optional to required_args ? -# required_args.add_argument("-e", "--environment", type=str, nargs='+', -# help="the environment (either 'staging' or 'production')") -# optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") -# optional_args.add_argument("-c", "--check", type=str, nargs='+', -# help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") -# -# args = parser.parse_args() -# -# if not args.environment: -# args_error(parser, "You have to provide an environment. Either 'staging' or 'production'") -# if not args.environment[0] in ('staging', 'production'): -# args_error(parser, "The environment has to be either 'staging' or 'production'") -# if len(args.environment) > 1: -# args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") -# -# if not args.tenantid: -# args.tenantid = [''] -# if not args.check: -# args.check = ['all'] -# -# return args.environment[0], args.tenantid[0], args.check[0] -# -# -# def read_yaml_file(path): -# """ -# reads a .yaml file and returns a dictionary -# :param path: path to the yaml file -# :return: returns a dictionary -# """ -# # ToDo error handling if path or file does not exist -# # FileNotFoundError: -# with open(path, 'r') as f: -# content = yaml.load(f, Loader=yaml.FullLoader) -# -# return content -# -# -# def parse_config(config, env_config): -# -# # ToDo check if "dummy" is really how it should be in the organizations file -# config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] -# # ToDo suche get all tenant funktion -# if not (hasattr(config,'tenant_urls') and config.tenant_urls): -# config.tenant_urls = {} -# for tenant_id in config.tenant_ids: -# config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) -# -# return True # ToDo return config ? +from parsing_configurations import __abort_script, log def check_users(tenant_id, digest_login, env_conf, config): - print('Log: start checking users for tenant ', tenant_id) + log('start checking users for tenant', tenant_id) external_api_accounts = {} for tenant in env_conf['opencast_organizations']: @@ -96,13 +37,6 @@ def get_roles_as_Json_array(account): return roles -# -# def generate_group_identifier(group, tenant_id): -# # ToDo move this to parse group config file -# # ToDo check if the generated identifiers are correct! (the same as in the ruby script) -# # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() -# return group['name'].replace(' ', '_').lower() - def create_user(account, digest_login, tenant_url): """ sends a POST request to the admin UI to create a User @@ -134,37 +68,3 @@ def create_user(account, digest_login, tenant_url): return False return response - - -# def get_groups_from_tenant(tenant_url, digest_login): -# -# url = '{}/api/groups/'.format(tenant_url) -# try: -# response = get_request(url, digest_login, '/api/groups/') -# except RequestError as err: -# print('RequestError:') -# print(err) -# return False -# except Exception as e: -# print("Groups could not be retrieved from {}. ".format(tenant_url)) -# print("Error: {}".format(str(e))) -# return False -# -# return response.json() - - -# def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): -# """ -# This function can be used to transform a json file to a yaml file. -# requires import json and import yaml -# :param json_file_path: path to json file -# :param yaml_file_path: path to yaml file (will be created if it does not exist) -# :return: -# """ -# -# with open(json_file_path, 'r') as json_file: -# jsonData = json.load(json_file) -# with open(yaml_file_path, 'w') as file: -# yaml.dump(jsonData, file, sort_keys=False) -# -# return True \ No newline at end of file diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 3f46e74..73b72b5 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -17,8 +17,6 @@ import config -VERBOSE_FLAG = True - def main(): ### Parse args and config ### @@ -39,137 +37,6 @@ def main(): check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) -# def check_users(tenant_id, digest_login, env_conf): -# print('Log: start checking users for tenant ', tenant_id) -# -# external_api_accounts = {} -# for tenant in env_conf['opencast_organizations']: -# id = tenant['id'] -# if id != "dummy": -# external_api_accounts[id] = tenant['external_api_accounts'] -# -# if not tenant_id: -# for_all_tenants = get_yes_no_answer("Create User for all tenants?") -# if not for_all_tenants: -# __abort_script("Okay, not doing anything.") -# else: -# # create user account for all tenants -# for tenant_id in config.tenant_ids: -# for account in external_api_accounts[tenant_id]: -# response = create_user(account, digest_login, config.tenant_urls[tenant_id]) -# else: -# # create user accounts on the specified tenant -# for account in external_api_accounts[tenant_id]: -# response = create_user(account, digest_login, config.tenant_urls[tenant_id]) - - -# def check_groups(tenant_id, digest_login, group_config): -# -# tenant_url = config.tenant_urls[tenant_id] -# # For all Groups: -# for group in group_config['groups']: -# if not tenant_id: -# tenant_id = group['tenants'] -# group['identifier'] = generate_group_identifier(group, tenant_id) -# # Check group -# if group['tenants'] == 'all' or group['tenants'] == tenant_id: -# check_group(tenant_url=tenant_url, digest_login=digest_login, group=group, tenant_id=tenant_id) - - -# def check_if_group_exists(tenant_url, digest_login, group, tenant_id): -# # ToDo log -# url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) -# try: -# response = get_request(url, digest_login, '/api/groups/') -# return response.json() -# except RequestError as err: -# if err.get_status_code() == "404": -# print('Group was not found: ', err) -# return False -# else: -# raise Exception -# except Exception as e: -# print("ERROR: {}".format(str(e))) -# return False - -# -# def check_group(tenant_url, digest_login, group, tenant_id): -# # ToDo log -# print(f"Checking group {group['name']} with id {group['identifier']}") -# -# # Check if group exists. -# existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) -# if not existing_group: -# # Create group if it does not exist. Ask for permission -# answer = get_yes_no_answer(f"group {group['name']} does not exist. Create group?") -# if answer: -# existing_group = create_group(group=group, digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id) -# elif existing_group: -# # Check if group name and description match the name and description provided in the configuration. -# print('check names:') -# if group['name'] == existing_group['name']: -# print('names are equal') -# # Update them if they do not match. (Asks for permission) -# # Check if group members exist. -# # Create missing group members. (Asks for permission) -# # Check if group roles match the group roles provided in the configuration. -# # Update group roles if they do not match.(Asks for permission) -# # Check if group members match the group members provided in the configuration. Add or remove members accordingly. -# # Check external API accounts of members. Add missing API accounts. -# # Check group type. If group is closed, remove unexpected members. -# # Update group members. (Asks for permission) - - -# def create_group(group, digest_login, tenant_url, tenant_id): -# # ToDo log -# print('Try to create Group!') -# url = '{}/api/groups/'.format(tenant_url) -# -# # ToDo is this logic correct? -# # should be checked if the member exists? -# members = [member['uid'] for member in group['members'] -# if member['tenants'] == 'all' or member['tenants'] == tenant_id] -# members = ",".join(members) -# # print("members: ", members) -# -# # ToDo check group config file if 'add' and 'remove' are needed -# roles = [] -# for permission in group['permissions']: -# if permission['tenants'] == 'all' or permission['tenants'] == tenant_id: -# for role in permission['roles']: -# roles.append(role) -# roles = ','.join(roles) -# # print("roles: ", roles) -# -# data = { -# 'name': group['name'], -# 'description': group['description'], -# 'roles': roles, -# 'members': members, -# } -# try: -# response = post_request(url, digest_login, '/api/groups/', data=data) -# print("created group {}".format(group['name'])) -# except RequestError as err: -# if err.get_status_code() == "400": -# print("Conflict: group with name {} could not be created.".format(group['name'])) -# elif err.get_status_code() == "409": -# print("Failed to create group: ", err) -# else: -# print(err) -# return False -# except Exception as e: -# print("Group could not be created: {}".format(str(e))) -# return False -# -# return response - - -def log(message): - if(VERBOSE_FLAG): - print(message) - - if __name__ == '__main__': try: main() diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 6a317dd..cc9aa9d 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -6,6 +6,8 @@ from rest_requests.request_error import RequestError +VERBOSE_FLAG = True + def parse_args(): """ Parse the arguments and check them for correctness @@ -21,6 +23,7 @@ def parse_args(): optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") optional_args.add_argument("-c", "--check", type=str, nargs='+', help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") + optional_args.add_argument("-v", "--verbose", type=str, nargs='+',help="enables more logging") args = parser.parse_args() @@ -31,10 +34,13 @@ def parse_args(): if len(args.environment) > 1: args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") - if not args.tenantid: - args.tenantid = [''] - if not args.check: - args.check = ['all'] + if not args.tenantid: args.tenantid = [''] + if not args.check: args.check = ['all'] + global VERBOSE_FLAG + if args.verbose[0] == "True": + VERBOSE_FLAG = True + else: + VERBOSE_FLAG = False return args.environment[0], args.tenantid[0], args.check[0] @@ -66,68 +72,6 @@ def parse_config(config, env_config): return config -# def get_roles_as_Json_array(account): -# roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] -# -# return roles - - -# def generate_group_identifier(group, tenant_id): -# # ToDo move this to parse group config file -# # ToDo check if the generated identifiers are correct! (the same as in the ruby script) -# # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() -# return group['name'].replace(' ', '_').lower() - -# -# def create_user(account, digest_login, tenant_url): -# """ sends a POST request to the admin UI to create a User -# -# :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} -# :param digest_login: digest login -# :param tenant_url: tenant url -# :return: -# """ -# url = '{}/admin-ng/users/'.format(tenant_url) -# data = { -# 'username': account['username'], -# 'password': account['password'], -# 'name': account['name'], -# 'email': account['email'], -# 'roles': str(get_roles_as_Json_array(account)) -# } -# try: -# response = post_request(url, digest_login, '/admin-ng/users/', data=data) -# print("created user {}".format(account['username'])) -# except RequestError as err: -# if err.get_status_code() == "409": -# print("Conflict, a user with username {} already exist.".format(account['username'])) -# if err.get_status_code() == "403": -# print("Forbidden, not enough permissions to create a user with a admin role.") -# return False -# except Exception as e: -# print("User could not be created: {}".format(str(e))) -# return False -# -# return response - - -# def get_groups_from_tenant(tenant_url, digest_login): -# -# url = '{}/api/groups/'.format(tenant_url) -# try: -# response = get_request(url, digest_login, '/api/groups/') -# except RequestError as err: -# print('RequestError:') -# print(err) -# return False -# except Exception as e: -# print("Groups could not be retrieved from {}. ".format(tenant_url)) -# print("Error: {}".format(str(e))) -# return False -# -# return response.json() - - def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): """ This function can be used to transform a json file to a yaml file. @@ -145,6 +89,11 @@ def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test return True +def log(*args): + if(VERBOSE_FLAG): + print(*args) + + def __abort_script(message): print(message) sys.exit() From 9c1854425fd754eba3dc11d18a5dd46f292b047f Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 17 May 2021 16:34:06 +0200 Subject: [PATCH 14/79] added function to send a PUT request to request.py --- lib/rest_requests/request.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lib/rest_requests/request.py b/lib/rest_requests/request.py index b9518a0..8b06131 100644 --- a/lib/rest_requests/request.py +++ b/lib/rest_requests/request.py @@ -129,3 +129,41 @@ def big_post_request(url, digest_login, element_description, asset_type_descript raise RequestError.with_status_code(url, str(response.status_code), element_description, asset_type_description, asset_description) return response + + +def put_request(url, digest_login, element_description, asset_type_description=None, asset_description=None, + data=None, files=None): + """ + Make a put request to the given url with the given digest login. If the request fails with an error or a status + code != 200, a Request Error with the error message /status code and the given descriptions is thrown. + + :param url: URL to make put request to + :type url: str + :param digest_login: The login credentials for digest authentication + :type digest_login: DigestLogin + :param element_description: Element description in case of errors, e.g. 'event', 'series', 'tenants' + :type element_description: str + :param asset_type_description: Asset type type description in case of errors, e.g. 'series', 'episode' + :type asset_type_description: str + :param asset_description: Asset description in case of errors, e.g. 'Dublin Core catalogs', 'ACL' + :type asset_description: str + :param data: Any data to attach to the request + :type data: dict + :param files: Any files to attach to the request + :type files: dict + :return: response + :raise RequestError: + """ + + auth = HTTPDigestAuth(digest_login.user, digest_login.password) + headers = {"X-Requested-Auth": "Digest"} + + try: + response = requests.put(url, auth=auth, headers=headers, data=data, files=files) + except Exception as e: + raise RequestError.with_error(url, str(e), element_description, asset_type_description, asset_description) + + if response.status_code < 200 or response.status_code > 299: + raise RequestError.with_status_code(url, str(response.status_code), element_description, asset_type_description, + asset_description) + return response From 10811123ee51e0791b041df981da092d68e36d97 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 17 May 2021 17:11:04 +0200 Subject: [PATCH 15/79] Added member and role checks for group. Added update_group and create_group function using the External API. Added more User Interactions to group checks. and overall code improvements: - fixed verbose flag - added usage of get_tenants function while parsing the config - added function get_user to configure_users.py --- multi-tenant-configuration/config.py | 8 +- .../configure_groups.py | 305 +++++++++++++++--- multi-tenant-configuration/configure_users.py | 37 ++- multi-tenant-configuration/main.py | 8 +- .../parsing_configurations.py | 16 +- 5 files changed, 314 insertions(+), 60 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 14c449d..93e2330 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -1,12 +1,12 @@ # Configuration # Set this to your global admin node -url = "http://tenant1:8080" +base_url = "http://localhost:8080" -# If you have multiple tenants use an URL pattern: -# tenant_url_pattern = "https://{}.example.org" -# ToDo otherwise, this can be empty or commented out +# If you have multiple tenants use an URL pattern. # ToDo otherwise, this can be empty or commented out +# example: tenant_url_pattern = "https://{}.example.org" tenant_url_pattern = "http://{}:8080" + # ToDo You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: # # example: # tenant_urls = { '': 'http://tenant1:8080', '': 'http://tenant2:8080' } diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 5ec9970..49c36cf 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -2,33 +2,36 @@ import json from args.args_parser import get_args_parser from args.args_error import args_error -from rest_requests.request import get_request, post_request +from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError -from input_output.input import get_yes_no_answer +from configure_users import get_user +from input_output.input import get_yes_no_answer, get_configurable_answer from parsing_configurations import log def check_groups(tenant_id, digest_login, group_config, config): + log('\nstart checking groups for tenant: ', tenant_id) tenant_url = config.tenant_urls[tenant_id] # For all Groups: for group in group_config['groups']: + # check for all tenants if tenant_id is not given if not tenant_id: tenant_id = group['tenants'] - group['identifier'] = generate_group_identifier(group, tenant_id) # Check group if group['tenants'] == 'all' or group['tenants'] == tenant_id: + group['identifier'] = generate_group_identifier(group, tenant_id) check_group(tenant_url=tenant_url, digest_login=digest_login, group=group, tenant_id=tenant_id) def check_if_group_exists(tenant_url, digest_login, group, tenant_id): - # ToDo log + log(f"check if group {group['name']} exists ...") + url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) try: response = get_request(url, digest_login, '/api/groups/') return response.json() except RequestError as err: if err.get_status_code() == "404": - print('Group was not found: ', err) return False else: raise Exception @@ -38,33 +41,179 @@ def check_if_group_exists(tenant_url, digest_login, group, tenant_id): def check_group(tenant_url, digest_login, group, tenant_id): - log(f"Check group {group['name']} with id {group['identifier']}") + log(f"\nCheck group {group['name']} with id {group['identifier']}") # Check if group exists. existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) if not existing_group: # Create group if it does not exist. Ask for permission - answer = get_yes_no_answer(f"group {group['name']} does not exist. Create group?") + answer = get_yes_no_answer(f"Group {group['name']} does not exist. Create group?") if answer: - existing_group = create_group(group=group, digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id) - elif existing_group: + create_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, group=group) + else: # Check if group name and description match the name and description provided in the configuration. - log('check names for group') - if group['name'] == existing_group['name']: - log('names are equal') # Update them if they do not match. (Asks for permission) + check_group_description(tenant_url=tenant_url, digest_login=digest_login, + group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. # Create missing group members. (Asks for permission) + check_group_members(tenant_url=tenant_url, digest_login=digest_login, + group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. # Update group roles if they do not match.(Asks for permission) + check_group_roles(tenant_url=tenant_url, digest_login=digest_login, + group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members match the group members provided in the configuration. Add or remove members accordingly. # Check external API accounts of members. Add missing API accounts. # Check group type. If group is closed, remove unexpected members. # Update group members. (Asks for permission) +def check_group_description(tenant_url, digest_login, group, existing_group, tenant_id): + log(f"check names and description for group {group['name']}.") + # ToDo: does it really makes sense to check for the name? + # This seems to be already done when checking for the existence of the group. + if group['name'] != existing_group['name']: + print("WARNING: Group names do not match. ") + return + if group_description_template(group['description'], tenant_id) == existing_group['description']: + log('Group descriptions match.') + else: + answer = get_yes_no_answer(f"Update group description for group {group['name']}?") + if answer: + update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, + description=group['description'], name=group['name']) + return + + +def check_group_members(tenant_url, digest_login, group, existing_group, tenant_id): + log(f"Check members for group {group['name']}.") + + group_members = extract_members_from_group(group=group, tenant_id=tenant_id).split(",") + existing_group_members = sorted(existing_group['members'].split(",")) + log("Config group members: ", group_members) + log("Existing group members: ", existing_group_members) + + if group_members == existing_group_members: + log('Group members match.') + else: + members = existing_group_members + missing_members = [member for member in group_members if member not in existing_group_members] + # check if missing members exist on the tenant + for member in missing_members: + if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): + print(f"Member {member} of group {group['name']} not found on tenant {tenant_id}.") + missing_members.remove(member) + additional_members = [member for member in existing_group_members if member not in group_members] + print("Missing members: ", missing_members) + print("Additional members: ", additional_members) + + if missing_members or additional_members: + update_answer = get_configurable_answer( + options=['y', 'n', 'a', 'r'], + short_descriptions=["Yes", "No", "Add missing members", "Remove additional members"], + long_descriptions=["updating group members", "skipping group", + "only adding missing members", "only removing additional members"], + question=f"Group members for group {group['name']} do not match. Update group?\n" + ) + if missing_members and update_answer in ['y', 'a']: + answer = get_configurable_answer( + options=['y', 'n', 'i'], + short_descriptions=["Yes, all", "No, none", "individual"], + long_descriptions=["adding missing members", "skipping missing members", "selecting individually"], + question=f"Add missing group members from the config file to group {group['name']}?" + ) + if answer == 'y': + members += missing_members + elif answer == 'i': + for member in missing_members: + answer = get_yes_no_answer(f"Add member {member} to group {group['name']}?") + if answer: + members.append(member) + + if additional_members and additional_members[0] and update_answer in ['y', 'r']: + answer = get_configurable_answer( + options=['y', 'n', 'i'], + short_descriptions=["Yes, all", "No, none", "individual"], + long_descriptions=["removing all additional members", + "keeping additional members", "selecting individually"], + question=f"Remove group members which are not in the config file from group {group['name']}?" + ) + if answer == 'y': + members -= additional_members + elif answer == 'i': + for member in additional_members: + answer = get_yes_no_answer(f"remove member {member} from group {group['name']}?") + if answer: + members.remove(member) + + members = ",".join(list(dict.fromkeys(members))) + update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, + members=members, name=group['name']) + + +def check_group_roles(tenant_url, digest_login, group, existing_group, tenant_id): + log(f"Check roles for group {group['name']}.") + + group_roles = extract_roles_from_group(group=group, tenant_id=tenant_id).split(",") + existing_group_roles = sorted(existing_group['roles'].split(",")) + log("Config group roles: ", group_roles) + log("Existing group roles: ", existing_group_roles) + + if group_roles == existing_group_roles: + log('Group roles match.') + else: + roles = existing_group_roles + missing_roles = [role for role in group_roles if role not in existing_group_roles] + additional_roles = [role for role in existing_group_roles if role not in group_roles] + print("Missing roles: ", missing_roles) + print("Additional roles: ", additional_roles) + + update_answer = get_configurable_answer( + options=['y', 'n', 'a', 'r'], + short_descriptions=["Yes", "No", "Add missing roles", "Remove additional roles"], + long_descriptions=["updating group roles", "skipping group", + "only adding missing roles", "only removing additional roles"], + question=f"Group roles for group {group['name']} do not match. Update group?\n" + ) + if missing_roles and update_answer in ['y', 'a']: + answer = get_configurable_answer( + options=['y', 'n', 'i'], + short_descriptions=["Yes, all", "No, none", "individual"], + long_descriptions=["adding missing roles", "skipping missing roles", "selecting individually"], + question=f"Add missing group roles from the config file to group {group['name']}?\n" + ) + if answer == 'y': + roles += missing_roles + elif answer == 'i': + for role in missing_roles: + answer = get_yes_no_answer(f"Add role {role} to group {group['name']}?") + if answer: + roles.append(role) + + if additional_roles and update_answer in ['y', 'r']: + answer = get_configurable_answer( + options=['y', 'n', 'i'], + short_descriptions=["Yes, all", "No, none", "individual"], + long_descriptions=["removing all additional roles", + "keeping additional roles", "selecting individually"], + question=f"Remove group roles which are not in the config file from group {group['name']}?" + ) + if answer == 'y': + roles -= additional_roles + elif answer == 'i': + for role in additional_roles: + answer = get_yes_no_answer(f"remove role {role} from group {group['name']}?") + if answer: + roles.remove(role) + + print("roles: ", roles) + roles = ",".join(list(dict.fromkeys(roles))) + update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, + roles=roles, name=group['name']) + + def generate_group_identifier(group, tenant_id): - # ToDo move this to parse group config file # ToDo check if the generated identifiers are correct! (the same as in the ruby script) # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() return group['name'].replace(' ', '_').lower() @@ -76,57 +225,135 @@ def get_groups_from_tenant(tenant_url, digest_login): try: response = get_request(url, digest_login, '/api/groups/') except RequestError as err: - print('RequestError:') - print(err) + print('RequestError: ', err) return False except Exception as e: - print("Groups could not be retrieved from {}. ".format(tenant_url)) - print("Error: {}".format(str(e))) + print(f"Groups could not be retrieved from {tenant_url}. \n", "Error: ", str(e)) return False return response.json() -def create_group(group, digest_login, tenant_url, tenant_id): - # ToDo log - print('Try to create Group!') - url = '{}/api/groups/'.format(tenant_url) - - # ToDo is this logic correct? - # should be checked if the member exists? - members = [member['uid'] for member in group['members'] - if member['tenants'] == 'all' or member['tenants'] == tenant_id] - members = ",".join(members) - # print("members: ", members) +def extract_roles_from_group(group, tenant_id): + """ - # ToDo check group config file if 'add' and 'remove' are needed + :param group: + :param tenant_id: + :return: sorted comma separated list of roles (e.g. "ROLE_ADMIN,ROLE_SUDO") + """ roles = [] for permission in group['permissions']: - if permission['tenants'] == 'all' or permission['tenants'] == tenant_id: + # add all default roles + if permission['tenants'] == 'all': for role in permission['roles']: roles.append(role) - roles = ','.join(roles) - # print("roles: ", roles) + # add/remove tenant specific roles + elif permission['tenants'] == tenant_id: + for role in permission['roles']['add']: + roles.append(role) + for role in permission['roles']['remove']: + if role in roles: + roles.remove(role) + roles = ','.join(sorted(roles)) + return roles + + +def extract_members_from_group(group, tenant_id): + """ + Does not check if member exists on tenant + :param group: + :param tenant_id: + :return: Comma separated list of members (e.g. "guy1,guy2") + """ + members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] + members = ",".join(sorted(members)) + return members + + +def group_description_template(description, tenant_id): + # ToDo check for a better way to insert into template + description = description.replace("${name}", tenant_id) + + return description + + +def update_group(digest_login, tenant_url, tenant_id, + group=None, name=None, description=None, roles=None, members=None): + log(f"Try to update group ... ") + if not name and not group: + log("Cannot update group without a specified name.") + return False + + if group: + group_id = group['identifier'] + if not name: + name = group['name'] + if not members: + members = extract_members_from_group(group, tenant_id) + if not roles: + roles = extract_roles_from_group(group, tenant_id) + if not description: + description = group_description_template(group['description'], tenant_id) + else: + group_id = generate_group_identifier(group={'name': name}, tenant_id=tenant_id) + url = f'{tenant_url}/api/groups/{group_id}' + + data = { + 'name': name, + 'description': description, + 'roles': roles, + 'members': members, + } + print('data ', data) + try: + response = put_request(url, digest_login, '/api/groups/{groupId}', data=data) + except RequestError as err: + if err.get_status_code() == "400": # ToDo: check if this is actually 404 + print(f"Bad Request: Group with name {name} does not exist.") + print("RequestError: ", err) + return False + except Exception as e: + print(f"Group with name {name} could not be updated. \n", "Exception: ", str(e)) + return False + + log(f"Updated group {name}.") + return response + + +def create_group(digest_login, tenant_url, tenant_id, group): + log(f"Try to create group {group['name']} ... ") + + url = f'{tenant_url}/api/groups/' + # extract members and roles + members = extract_members_from_group(group, tenant_id).split(",") + # check if member exist on tenant + for member in members: + if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): + print(f"Member {member} does not exist.") + members.remove(member) + members = ",".join(members) + roles = extract_roles_from_group(group, tenant_id) + description = group_description_template(group['description'], tenant_id) data = { 'name': group['name'], - 'description': group['description'], + 'description': description, 'roles': roles, 'members': members, } try: response = post_request(url, digest_login, '/api/groups/', data=data) - print("created group {}".format(group['name'])) except RequestError as err: if err.get_status_code() == "400": - print("Conflict: group with name {} could not be created.".format(group['name'])) + print(f"Bad Request: Group with name {group['name']} could not be created.") elif err.get_status_code() == "409": - print("Failed to create group: ", err) - else: - print(err) + print(f"Conflict: Group with name {group['name']} could not be created.\n" + f"Potentially, Group with name {group['name']} already exists.") + print("RequestError: ", err) return False except Exception as e: - print("Group could not be created: {}".format(str(e))) + print(f"Group with name {group['name']} could not be created. \n", "Exception: ", str(e)) return False + log(f"created group {group['name']}.\nmembers: {members} \nroles: {roles} ") return response diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index a4697b4..3fbb83c 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -9,11 +9,12 @@ def check_users(tenant_id, digest_login, env_conf, config): - log('start checking users for tenant', tenant_id) + log('\nstart checking users for tenant: ', tenant_id) external_api_accounts = {} for tenant in env_conf['opencast_organizations']: id = tenant['id'] + # ToDo check if this is necessary if id != "dummy": external_api_accounts[id] = tenant['external_api_accounts'] @@ -29,7 +30,7 @@ def check_users(tenant_id, digest_login, env_conf, config): else: # create user accounts on the specified tenant for account in external_api_accounts[tenant_id]: - response = create_user(account, digest_login, config.tenant_urls[tenant_id]) + create_user(account, digest_login, config.tenant_urls[tenant_id]) def get_roles_as_Json_array(account): @@ -43,16 +44,16 @@ def create_user(account, digest_login, tenant_url): :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} :param digest_login: digest login - :param tenant_url: tenant url + :param tenant_url: tenant url :return: """ url = '{}/admin-ng/users/'.format(tenant_url) data = { 'username': account['username'], 'password': account['password'], - 'name': account['name'], - 'email': account['email'], - 'roles': str(get_roles_as_Json_array(account)) + 'name': account['name'], + 'email': account['email'], + 'roles': str(get_roles_as_Json_array(account)) } try: response = post_request(url, digest_login, '/admin-ng/users/', data=data) @@ -68,3 +69,27 @@ def create_user(account, digest_login, tenant_url): return False return response + + +def get_user(username, digest_login, tenant_url): + """ sends a GET request to the admin UI to get a User + + :param username: String + :param digest_login: digest login + :param tenant_url: tenant url + :return: + """ + url = f'{tenant_url}/admin-ng/users/{username}.json' + try: + response = get_request(url, digest_login, '/admin-ng/users/{username}.json') + except RequestError as err: + if err.get_status_code() == "404": + return False + else: + print(err) + return False + except Exception as e: + print(e) + return False + + return response diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 73b72b5..afad8e9 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -20,12 +20,12 @@ def main(): ### Parse args and config ### - environment, tenant_id, check = parse_args() # parse args - env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file - script_config = parse_config(config, env_conf) # parse config.py - group_config = read_yaml_file(config.group_path) # read group config file # ToDo Think about whether we should exclude Digest Login credentials from config.py file digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # create Digest Login + environment, tenant_id, check = parse_args() # parse args + env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file + script_config = parse_config(config, env_conf, digest_login) # parse config.py + group_config = read_yaml_file(script_config.group_path) # read group config file ### Start checks ### if check == 'all': diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index cc9aa9d..3074a78 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -4,6 +4,7 @@ from args.args_error import args_error from rest_requests.request import get_request, post_request from rest_requests.request_error import RequestError +from rest_requests.basic_requests import get_tenants VERBOSE_FLAG = True @@ -37,7 +38,7 @@ def parse_args(): if not args.tenantid: args.tenantid = [''] if not args.check: args.check = ['all'] global VERBOSE_FLAG - if args.verbose[0] == "True": + if args.verbose and args.verbose[0] == "True": VERBOSE_FLAG = True else: VERBOSE_FLAG = False @@ -59,12 +60,13 @@ def read_yaml_file(path): return content -def parse_config(config, env_config): +def parse_config(config, env_config, digest_login): + # ToDo Check if all mandatory configurations are given - # ToDo check if "dummy" is really how it should be in the organizations file - config.tenant_ids = [tenant['id'] for tenant in env_config['opencast_organizations'] if tenant['id'] != "dummy"] - # ToDo suche get all tenant funktion - if not (hasattr(config,'tenant_urls') and config.tenant_urls): + # ToDo should mh_default_org be removed from tenant_ids? + config.tenant_ids = get_tenants(config.base_url, digest_login) + + if not (hasattr(config, 'tenant_urls') and config.tenant_urls): config.tenant_urls = {} for tenant_id in config.tenant_ids: config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) @@ -90,7 +92,7 @@ def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test def log(*args): - if(VERBOSE_FLAG): + if VERBOSE_FLAG: print(*args) From 97d92240754e3d376fe56e6707943c1456f0e4e9 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 7 Jun 2021 12:55:25 +0200 Subject: [PATCH 16/79] Updated User Interaction To improve the usability,the User can now store his answers by adding a 't' or 'a' to his answers. The question for that action will then not be asked again. --- .../user_interaction.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 multi-tenant-configuration/user_interaction.py diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py new file mode 100644 index 0000000..d9639c1 --- /dev/null +++ b/multi-tenant-configuration/user_interaction.py @@ -0,0 +1,124 @@ +from parsing_configurations import log +from collections import namedtuple +import re + + +Permission = namedtuple('Permission', ['tenant', 'target', 'permission_value']) +permissions = { + 'user': {}, + 'group': {} +} +ANSWER_PATTERN = r"^[yn]$|^[yn][ta][ta]$" +HELP_OPTION = 'h' + + +def check_or_ask_for_permission(target_type, action, target_name, tenant_id, option_i=False) -> bool: + + # check if permission is already defined + permission = get_permission(target_type, action, target_name, tenant_id) + if permission is None: + # otherwise ask for user input + answer = ask_user(target_type, action, target_name, tenant_id, option_i) + # process answer and update permissions + permission = process_answer(answer, target_type, action, target_name, tenant_id, option_i) + + return permission + + +def get_permission(target_type, action, target_name, tenant_id) -> bool: + + log('permissions: ', permissions) + + permission = None + target_permission = None + tenant_permission = None + + try: + for p in permissions[target_type][action]: + # most specific permission + if p.tenant == tenant_id and p.target == target_name: + permission = p.permission_value + break + # either tenant or target specific + elif p.tenant == 'all' and p.target == target_name: + target_permission = p.permission_value + elif p.tenant == tenant_id and p.target == 'all': + tenant_permission = p.permission_value + # most general permission + elif p.tenant == 'all' and p.target == 'all': + permission = p.permission_value + + # target permission is prioritized over tenant permission + # both will overwrite a general permission + if target_permission is not None: + permission = target_permission + elif tenant_permission is not None: + permission = tenant_permission + + # if no permission is found, None is returned + except KeyError: + print('no permission found') + + return permission + + +def ask_user(target_type, action, target_name, tenant_id, option_i=False) -> str: + + help_description = "Valid answers are: \nHELP DESCRIPTION" + + individual_option = "\n Write 'i' to perform the action individually for each case. " if option_i else "" + + question = f"""Do you want to {action} ({target_type} {target_name} on {tenant_id})? + Write 'y' to perform the action. Write 'n' to skipp this action. {individual_option}Write '{HELP_OPTION}' for help. + Add 't' for 'tenant' or 'a' for 'all' to store your decision for this or all tenants. + Add 't' for 'target' or 'a' for 'all' to store your decision for this or all targets. + EXAMPLE: Write 'yat' to store your decision for the action on ALL tenants and for THIS target. +""" + + answer = '' + while True: + # catch the help option: give a more detailed description of the options + if answer == HELP_OPTION: + answer = input(help_description) + # ask the question + else: + answer = input(question).lower() + # return all valid answers + if parsable(answer) or (option_i and answer == 'i'): + return answer + else: + print("Invalid answer.\n") + + +def parsable(answer) -> bool: + + if re.match(ANSWER_PATTERN, answer) or answer == HELP_OPTION: + return True + else: + return False + + +def process_answer(answer, target_type, action, target_name, tenant_id, option_i) -> bool: + + # simple yes or no case (not stored) + if answer == 'y': + return True + if answer == 'n': + return False + # individual case + if option_i and answer == 'i': + return answer + + # store answer if user specified this + permission_value = True if answer.startswith('y') else False + tenant = 'all' if answer[1] == 'a' else tenant_id + target = 'all' if answer[2] == 'a' else target_name + p = Permission(tenant, target, permission_value) + + try: + permissions[target_type][action].append(p) + except: + permissions[target_type][action] = [p] + + return permission_value + From cd776b045b6edc92ccda6cc37056bd0d7b978bfb Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 7 Jun 2021 17:09:24 +0200 Subject: [PATCH 17/79] Integrated improved User interaction for group checks Improved User questions included for member and roles checks. --- .../configure_groups.py | 228 ++++++++++-------- 1 file changed, 125 insertions(+), 103 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 49c36cf..8cb555a 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -6,10 +6,12 @@ from rest_requests.request_error import RequestError from configure_users import get_user from input_output.input import get_yes_no_answer, get_configurable_answer +from user_interaction import check_or_ask_for_permission from parsing_configurations import log def check_groups(tenant_id, digest_login, group_config, config): log('\nstart checking groups for tenant: ', tenant_id) + # ToDo handle case if no tenant_id is given tenant_url = config.tenant_urls[tenant_id] # For all Groups: @@ -47,8 +49,13 @@ def check_group(tenant_url, digest_login, group, tenant_id): existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) if not existing_group: # Create group if it does not exist. Ask for permission - answer = get_yes_no_answer(f"Group {group['name']} does not exist. Create group?") - if answer: + action_allowed = check_or_ask_for_permission( + target_type='group', + action='create', + target_name=group['name'], + tenant_id=tenant_id + ) + if action_allowed: create_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, group=group) else: # Check if group name and description match the name and description provided in the configuration. @@ -57,13 +64,14 @@ def check_group(tenant_url, digest_login, group, tenant_id): group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. # Create missing group members. (Asks for permission) + # Check if group members match the group members provided in the configuration. Add or remove members accordingly. check_group_members(tenant_url=tenant_url, digest_login=digest_login, group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. # Update group roles if they do not match.(Asks for permission) check_group_roles(tenant_url=tenant_url, digest_login=digest_login, group=group, existing_group=existing_group, tenant_id=tenant_id) - # Check if group members match the group members provided in the configuration. Add or remove members accordingly. + # Check external API accounts of members. Add missing API accounts. # Check group type. If group is closed, remove unexpected members. # Update group members. (Asks for permission) @@ -79,77 +87,88 @@ def check_group_description(tenant_url, digest_login, group, existing_group, ten if group_description_template(group['description'], tenant_id) == existing_group['description']: log('Group descriptions match.') else: - answer = get_yes_no_answer(f"Update group description for group {group['name']}?") - if answer: - update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, - description=group['description'], name=group['name']) + action_allowed = check_or_ask_for_permission( + target_type='group', + action='update the description', + target_name=group['name'], + tenant_id=tenant_id + ) + if action_allowed: + update_group( + digest_login=digest_login, + tenant_url=tenant_url, + tenant_id=tenant_id, + description=group['description'], + name=group['name'] + ) + return def check_group_members(tenant_url, digest_login, group, existing_group, tenant_id): log(f"Check members for group {group['name']}.") - group_members = extract_members_from_group(group=group, tenant_id=tenant_id).split(",") - existing_group_members = sorted(existing_group['members'].split(",")) + group_members = extract_members_from_group(group=group, tenant_id=tenant_id) + existing_group_members = sorted(filter(None, existing_group['members'].split(","))) + log("Config group members: ", group_members) log("Existing group members: ", existing_group_members) - if group_members == existing_group_members: + members = existing_group_members.copy() + missing_members = [member for member in group_members if member not in existing_group_members] + for member in missing_members: + if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): + log(f"Member {member} of group {group['name']} not found on tenant {tenant_id}.") + missing_members.remove(member) + additional_members = [member for member in existing_group_members if member not in group_members] + + if not missing_members and not additional_members: log('Group members match.') else: - members = existing_group_members - missing_members = [member for member in group_members if member not in existing_group_members] - # check if missing members exist on the tenant - for member in missing_members: - if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): - print(f"Member {member} of group {group['name']} not found on tenant {tenant_id}.") - missing_members.remove(member) - additional_members = [member for member in existing_group_members if member not in group_members] - print("Missing members: ", missing_members) - print("Additional members: ", additional_members) - - if missing_members or additional_members: - update_answer = get_configurable_answer( - options=['y', 'n', 'a', 'r'], - short_descriptions=["Yes", "No", "Add missing members", "Remove additional members"], - long_descriptions=["updating group members", "skipping group", - "only adding missing members", "only removing additional members"], - question=f"Group members for group {group['name']} do not match. Update group?\n" - ) - if missing_members and update_answer in ['y', 'a']: - answer = get_configurable_answer( - options=['y', 'n', 'i'], - short_descriptions=["Yes, all", "No, none", "individual"], - long_descriptions=["adding missing members", "skipping missing members", "selecting individually"], - question=f"Add missing group members from the config file to group {group['name']}?" + if missing_members: + print("Missing members: ", missing_members) + action_allowed = check_or_ask_for_permission( + target_type='group', + action='add missing members', + target_name=group['name'], + tenant_id=tenant_id, + option_i=True ) - if answer == 'y': - members += missing_members - elif answer == 'i': + if action_allowed == 'i': for member in missing_members: - answer = get_yes_no_answer(f"Add member {member} to group {group['name']}?") - if answer: + action_allowed = get_yes_no_answer(f"Add member {member} to group {group['name']}?") + if action_allowed: members.append(member) - - if additional_members and additional_members[0] and update_answer in ['y', 'r']: - answer = get_configurable_answer( - options=['y', 'n', 'i'], - short_descriptions=["Yes, all", "No, none", "individual"], - long_descriptions=["removing all additional members", - "keeping additional members", "selecting individually"], - question=f"Remove group members which are not in the config file from group {group['name']}?" + elif action_allowed: + for member in missing_members: + members.append(member) + + if additional_members: + print("Additional members: ", additional_members) + action_allowed = check_or_ask_for_permission( + target_type='group', + action='remove additional members', + target_name=group['name'], + tenant_id=tenant_id, + option_i=True ) - if answer == 'y': - members -= additional_members - elif answer == 'i': + if action_allowed == 'i': for member in additional_members: - answer = get_yes_no_answer(f"remove member {member} from group {group['name']}?") - if answer: + action_allowed = get_yes_no_answer(f"remove member {member} from group {group['name']}?") + if action_allowed: members.remove(member) + elif action_allowed: + for member in additional_members: + members.remove(member) - members = ",".join(list(dict.fromkeys(members))) - update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, - members=members, name=group['name']) + # Update Group if there are any changes + if members != existing_group_members: + # members = ",".join(list(dict.fromkeys(members))) + members = ",".join(members) + update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, + group=group, members=members) + + return def check_group_roles(tenant_url, digest_login, group, existing_group, tenant_id): @@ -157,60 +176,60 @@ def check_group_roles(tenant_url, digest_login, group, existing_group, tenant_id group_roles = extract_roles_from_group(group=group, tenant_id=tenant_id).split(",") existing_group_roles = sorted(existing_group['roles'].split(",")) + log("Config group roles: ", group_roles) log("Existing group roles: ", existing_group_roles) + roles = existing_group_roles.copy() + missing_roles = [role for role in group_roles if role not in existing_group_roles] + additional_roles = [role for role in existing_group_roles if role not in group_roles] + if group_roles == existing_group_roles: log('Group roles match.') else: - roles = existing_group_roles - missing_roles = [role for role in group_roles if role not in existing_group_roles] - additional_roles = [role for role in existing_group_roles if role not in group_roles] - print("Missing roles: ", missing_roles) - print("Additional roles: ", additional_roles) - - update_answer = get_configurable_answer( - options=['y', 'n', 'a', 'r'], - short_descriptions=["Yes", "No", "Add missing roles", "Remove additional roles"], - long_descriptions=["updating group roles", "skipping group", - "only adding missing roles", "only removing additional roles"], - question=f"Group roles for group {group['name']} do not match. Update group?\n" - ) - if missing_roles and update_answer in ['y', 'a']: - answer = get_configurable_answer( - options=['y', 'n', 'i'], - short_descriptions=["Yes, all", "No, none", "individual"], - long_descriptions=["adding missing roles", "skipping missing roles", "selecting individually"], - question=f"Add missing group roles from the config file to group {group['name']}?\n" + if missing_roles: + print("Missing roles: ", missing_roles) + action_allowed = check_or_ask_for_permission( + target_type='group', + action='add missing group roles', + target_name=group['name'], + tenant_id=tenant_id, + option_i=True ) - if answer == 'y': - roles += missing_roles - elif answer == 'i': + if action_allowed == 'i': for role in missing_roles: - answer = get_yes_no_answer(f"Add role {role} to group {group['name']}?") - if answer: + action_allowed = get_yes_no_answer(f"Add role {role} to group {group['name']}?") + if action_allowed: roles.append(role) - - if additional_roles and update_answer in ['y', 'r']: - answer = get_configurable_answer( - options=['y', 'n', 'i'], - short_descriptions=["Yes, all", "No, none", "individual"], - long_descriptions=["removing all additional roles", - "keeping additional roles", "selecting individually"], - question=f"Remove group roles which are not in the config file from group {group['name']}?" + elif action_allowed: + for role in missing_roles: + roles.append(role) + + if additional_roles: + print("Additional roles: ", additional_roles) + action_allowed = check_or_ask_for_permission( + target_type='group', + action='remove additional group roles', + target_name=group['name'], + tenant_id=tenant_id, + option_i=True ) - if answer == 'y': - roles -= additional_roles - elif answer == 'i': + if action_allowed == 'i': for role in additional_roles: - answer = get_yes_no_answer(f"remove role {role} from group {group['name']}?") - if answer: + action_allowed = get_yes_no_answer(f"remove role {role} from group {group['name']}?") + if action_allowed: roles.remove(role) + elif action_allowed: + for role in additional_roles: + roles.remove(role) - print("roles: ", roles) - roles = ",".join(list(dict.fromkeys(roles))) - update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, - roles=roles, name=group['name']) + if roles != existing_group_roles: + # roles = ",".join(list(dict.fromkeys(roles))) + roles = ",".join(roles) + update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, + group=group, roles=roles) + + return def generate_group_identifier(group, tenant_id): @@ -258,15 +277,17 @@ def extract_roles_from_group(group, tenant_id): return roles -def extract_members_from_group(group, tenant_id): +def extract_members_from_group(group, tenant_id, as_string=False): """ Does not check if member exists on tenant :param group: :param tenant_id: - :return: Comma separated list of members (e.g. "guy1,guy2") + :param as_string: + :return: Comma separated string of members (e.g. "guy1,guy2") or list of members. """ members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] - members = ",".join(sorted(members)) + if as_string: + members = ",".join(sorted(members)) return members @@ -290,7 +311,7 @@ def update_group(digest_login, tenant_url, tenant_id, if not name: name = group['name'] if not members: - members = extract_members_from_group(group, tenant_id) + members = extract_members_from_group(group, tenant_id, as_string=True) if not roles: roles = extract_roles_from_group(group, tenant_id) if not description: @@ -322,11 +343,11 @@ def update_group(digest_login, tenant_url, tenant_id, def create_group(digest_login, tenant_url, tenant_id, group): - log(f"Try to create group {group['name']} ... ") + log(f"trying to create group {group['name']}. ") url = f'{tenant_url}/api/groups/' # extract members and roles - members = extract_members_from_group(group, tenant_id).split(",") + members = extract_members_from_group(group, tenant_id) # check if member exist on tenant for member in members: if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): @@ -341,6 +362,7 @@ def create_group(digest_login, tenant_url, tenant_id, group): 'roles': roles, 'members': members, } + try: response = post_request(url, digest_login, '/api/groups/', data=data) except RequestError as err: From f94d37b914c360ff5c38f22f574dba04e6eed2b9 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 11:14:58 +0200 Subject: [PATCH 18/79] Added function to perform checks for all tenants If the script is started without specifying a tenant_id it will now perform all checks for all tenants. --- .../configure_groups.py | 18 +++++++--------- multi-tenant-configuration/configure_users.py | 2 +- multi-tenant-configuration/main.py | 21 ++++++++++++------- .../parsing_configurations.py | 1 + 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 8cb555a..e4829bf 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -1,7 +1,7 @@ -import yaml -import json -from args.args_parser import get_args_parser -from args.args_error import args_error +# import yaml +# import json +# from args.args_parser import get_args_parser +# from args.args_error import args_error from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError from configure_users import get_user @@ -9,16 +9,13 @@ from user_interaction import check_or_ask_for_permission from parsing_configurations import log + def check_groups(tenant_id, digest_login, group_config, config): - log('\nstart checking groups for tenant: ', tenant_id) - # ToDo handle case if no tenant_id is given + log('\nStart checking groups for tenant: ', tenant_id) tenant_url = config.tenant_urls[tenant_id] # For all Groups: for group in group_config['groups']: - # check for all tenants if tenant_id is not given - if not tenant_id: - tenant_id = group['tenants'] # Check group if group['tenants'] == 'all' or group['tenants'] == tenant_id: group['identifier'] = generate_group_identifier(group, tenant_id) @@ -64,7 +61,8 @@ def check_group(tenant_url, digest_login, group, tenant_id): group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. # Create missing group members. (Asks for permission) - # Check if group members match the group members provided in the configuration. Add or remove members accordingly. + # Check if group members match the group members provided in the configuration. + # Add or remove members accordingly. check_group_members(tenant_url=tenant_url, digest_login=digest_login, group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 3fbb83c..3f9022f 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -9,7 +9,7 @@ def check_users(tenant_id, digest_login, env_conf, config): - log('\nstart checking users for tenant: ', tenant_id) + log('\nStart checking users for tenant: ', tenant_id) external_api_accounts = {} for tenant in env_conf['opencast_organizations']: diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index afad8e9..1433091 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -27,14 +27,21 @@ def main(): script_config = parse_config(config, env_conf, digest_login) # parse config.py group_config = read_yaml_file(script_config.group_path) # read group config file + # if tenant is not given, we perform the checks for all tenants + if tenant_id: + tenants_to_check = [tenant_id] + else: + tenants_to_check = script_config.tenant_ids + ### Start checks ### - if check == 'all': - check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) - check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) - elif check == 'users': - check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) - elif check == 'groups': - check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + for tenant_id in tenants_to_check: + if check == 'all': + check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) + check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + elif check == 'users': + check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) + elif check == 'groups': + check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) if __name__ == '__main__': diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 3074a78..a62ef2c 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -65,6 +65,7 @@ def parse_config(config, env_config, digest_login): # ToDo should mh_default_org be removed from tenant_ids? config.tenant_ids = get_tenants(config.base_url, digest_login) + config.tenant_ids.remove('mh_default_org') if not (hasattr(config, 'tenant_urls') and config.tenant_urls): config.tenant_urls = {} From 7c57358d45372df2737b87e1eadb0f80b39d7cd5 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 13:30:48 +0200 Subject: [PATCH 19/79] Refactoring code Added configurations and digest login as global variables. Now they don't have to be passed onto every function. --- .../configure_groups.py | 109 ++++++++++-------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index e4829bf..d7985fe 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -10,42 +10,38 @@ from parsing_configurations import log +CONFIG = None +GROUP_CONFIG = None +DIGEST_LOGIN = None + + def check_groups(tenant_id, digest_login, group_config, config): log('\nStart checking groups for tenant: ', tenant_id) - tenant_url = config.tenant_urls[tenant_id] + global DIGEST_LOGIN + global GROUP_CONFIG + global CONFIG + DIGEST_LOGIN = digest_login + GROUP_CONFIG = group_config + CONFIG = config + # tenant_url = CONFIG.tenant_urls[tenant_id] + # For all Groups: for group in group_config['groups']: # Check group if group['tenants'] == 'all' or group['tenants'] == tenant_id: group['identifier'] = generate_group_identifier(group, tenant_id) - check_group(tenant_url=tenant_url, digest_login=digest_login, group=group, tenant_id=tenant_id) - - -def check_if_group_exists(tenant_url, digest_login, group, tenant_id): - log(f"check if group {group['name']} exists ...") - - url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) - try: - response = get_request(url, digest_login, '/api/groups/') - return response.json() - except RequestError as err: - if err.get_status_code() == "404": - return False - else: - raise Exception - except Exception as e: - print("ERROR: {}".format(str(e))) - return False + check_group(group=group, tenant_id=tenant_id) -def check_group(tenant_url, digest_login, group, tenant_id): +def check_group(group, tenant_id): log(f"\nCheck group {group['name']} with id {group['identifier']}") # Check if group exists. - existing_group = check_if_group_exists(tenant_url, digest_login, group, tenant_id) + existing_group = check_if_group_exists(group, tenant_id) if not existing_group: - # Create group if it does not exist. Ask for permission + # Create group if it does not exist. + # Ask for permission action_allowed = check_or_ask_for_permission( target_type='group', action='create', @@ -53,29 +49,44 @@ def check_group(tenant_url, digest_login, group, tenant_id): tenant_id=tenant_id ) if action_allowed: - create_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, group=group) + create_group(group=group, tenant_id=tenant_id) else: # Check if group name and description match the name and description provided in the configuration. # Update them if they do not match. (Asks for permission) - check_group_description(tenant_url=tenant_url, digest_login=digest_login, - group=group, existing_group=existing_group, tenant_id=tenant_id) + check_group_description(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. - # Create missing group members. (Asks for permission) + # ToDo Create missing group members. (Asks for permission) ? # Check if group members match the group members provided in the configuration. # Add or remove members accordingly. - check_group_members(tenant_url=tenant_url, digest_login=digest_login, - group=group, existing_group=existing_group, tenant_id=tenant_id) + check_group_members(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. # Update group roles if they do not match.(Asks for permission) - check_group_roles(tenant_url=tenant_url, digest_login=digest_login, - group=group, existing_group=existing_group, tenant_id=tenant_id) + check_group_roles(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check external API accounts of members. Add missing API accounts. # Check group type. If group is closed, remove unexpected members. # Update group members. (Asks for permission) -def check_group_description(tenant_url, digest_login, group, existing_group, tenant_id): +def check_if_group_exists(group, tenant_id): + log(f"check if group {group['name']} exists.") + + tenant_url = CONFIG.tenant_urls[tenant_id] + url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) + try: + response = get_request(url, DIGEST_LOGIN, '/api/groups/') + return response.json() + except RequestError as err: + if err.get_status_code() == "404": + return False + else: + raise Exception + except Exception as e: + print("ERROR: {}".format(str(e))) + return False + + +def check_group_description(group, existing_group, tenant_id): log(f"check names and description for group {group['name']}.") # ToDo: does it really makes sense to check for the name? # This seems to be already done when checking for the existence of the group. @@ -93,8 +104,6 @@ def check_group_description(tenant_url, digest_login, group, existing_group, ten ) if action_allowed: update_group( - digest_login=digest_login, - tenant_url=tenant_url, tenant_id=tenant_id, description=group['description'], name=group['name'] @@ -103,9 +112,12 @@ def check_group_description(tenant_url, digest_login, group, existing_group, ten return -def check_group_members(tenant_url, digest_login, group, existing_group, tenant_id): +def check_group_members(group, existing_group, tenant_id): log(f"Check members for group {group['name']}.") + # ToDo remove this also from configure_users + tenant_url = CONFIG.tenant_urls[tenant_id] + group_members = extract_members_from_group(group=group, tenant_id=tenant_id) existing_group_members = sorted(filter(None, existing_group['members'].split(","))) @@ -115,7 +127,7 @@ def check_group_members(tenant_url, digest_login, group, existing_group, tenant_ members = existing_group_members.copy() missing_members = [member for member in group_members if member not in existing_group_members] for member in missing_members: - if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): + if not get_user(username=member, digest_login=DIGEST_LOGIN, tenant_url=tenant_url): log(f"Member {member} of group {group['name']} not found on tenant {tenant_id}.") missing_members.remove(member) additional_members = [member for member in existing_group_members if member not in group_members] @@ -163,13 +175,12 @@ def check_group_members(tenant_url, digest_login, group, existing_group, tenant_ if members != existing_group_members: # members = ",".join(list(dict.fromkeys(members))) members = ",".join(members) - update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, - group=group, members=members) + update_group(tenant_id=tenant_id, group=group, members=members) return -def check_group_roles(tenant_url, digest_login, group, existing_group, tenant_id): +def check_group_roles(group, existing_group, tenant_id): log(f"Check roles for group {group['name']}.") group_roles = extract_roles_from_group(group=group, tenant_id=tenant_id).split(",") @@ -224,8 +235,7 @@ def check_group_roles(tenant_url, digest_login, group, existing_group, tenant_id if roles != existing_group_roles: # roles = ",".join(list(dict.fromkeys(roles))) roles = ",".join(roles) - update_group(digest_login=digest_login, tenant_url=tenant_url, tenant_id=tenant_id, - group=group, roles=roles) + update_group(tenant_id=tenant_id, group=group, roles=roles) return @@ -236,11 +246,12 @@ def generate_group_identifier(group, tenant_id): return group['name'].replace(' ', '_').lower() -def get_groups_from_tenant(tenant_url, digest_login): +def get_groups_from_tenant(tenant_id): + tenant_url = CONFIG.tenant_urls[tenant_id] url = '{}/api/groups/'.format(tenant_url) try: - response = get_request(url, digest_login, '/api/groups/') + response = get_request(url, DIGEST_LOGIN, '/api/groups/') except RequestError as err: print('RequestError: ', err) return False @@ -297,8 +308,7 @@ def group_description_template(description, tenant_id): return description -def update_group(digest_login, tenant_url, tenant_id, - group=None, name=None, description=None, roles=None, members=None): +def update_group(tenant_id, group=None, name=None, description=None, roles=None, members=None): log(f"Try to update group ... ") if not name and not group: log("Cannot update group without a specified name.") @@ -316,6 +326,7 @@ def update_group(digest_login, tenant_url, tenant_id, description = group_description_template(group['description'], tenant_id) else: group_id = generate_group_identifier(group={'name': name}, tenant_id=tenant_id) + tenant_url = CONFIG.tenant_urls[tenant_id] url = f'{tenant_url}/api/groups/{group_id}' data = { @@ -326,7 +337,7 @@ def update_group(digest_login, tenant_url, tenant_id, } print('data ', data) try: - response = put_request(url, digest_login, '/api/groups/{groupId}', data=data) + response = put_request(url, DIGEST_LOGIN, '/api/groups/{groupId}', data=data) except RequestError as err: if err.get_status_code() == "400": # ToDo: check if this is actually 404 print(f"Bad Request: Group with name {name} does not exist.") @@ -340,15 +351,17 @@ def update_group(digest_login, tenant_url, tenant_id, return response -def create_group(digest_login, tenant_url, tenant_id, group): +def create_group(group, tenant_id): log(f"trying to create group {group['name']}. ") + tenant_url = CONFIG.tenant_urls[tenant_id] url = f'{tenant_url}/api/groups/' + # extract members and roles members = extract_members_from_group(group, tenant_id) # check if member exist on tenant for member in members: - if not get_user(username=member, digest_login=digest_login, tenant_url=tenant_url): + if not get_user(username=member, digest_login=DIGEST_LOGIN, tenant_url=tenant_url): print(f"Member {member} does not exist.") members.remove(member) members = ",".join(members) @@ -362,7 +375,7 @@ def create_group(digest_login, tenant_url, tenant_id, group): } try: - response = post_request(url, digest_login, '/api/groups/', data=data) + response = post_request(url, DIGEST_LOGIN, '/api/groups/', data=data) except RequestError as err: if err.get_status_code() == "400": print(f"Bad Request: Group with name {group['name']} could not be created.") From 396671343e8b7c953f1bc8fb70a8e97cd70163ef Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 14:04:45 +0200 Subject: [PATCH 20/79] Refactoring code Added configurations and digest login as global variables in configure_users.py . --- multi-tenant-configuration/configure_users.py | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 3f9022f..a6a1c91 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -1,18 +1,30 @@ -import yaml -import json -from args.args_parser import get_args_parser -from args.args_error import args_error +# import yaml +# import json +# from args.args_parser import get_args_parser +# from args.args_error import args_error from rest_requests.request import get_request, post_request from rest_requests.request_error import RequestError from input_output.input import get_yes_no_answer from parsing_configurations import __abort_script, log +CONFIG = None +ENV_CONFIG = None +DIGEST_LOGIN = None + + def check_users(tenant_id, digest_login, env_conf, config): log('\nStart checking users for tenant: ', tenant_id) + global DIGEST_LOGIN + global ENV_CONFIG + global CONFIG + DIGEST_LOGIN = digest_login + ENV_CONFIG = env_conf + CONFIG = config + external_api_accounts = {} - for tenant in env_conf['opencast_organizations']: + for tenant in ENV_CONFIG['opencast_organizations']: id = tenant['id'] # ToDo check if this is necessary if id != "dummy": @@ -24,40 +36,42 @@ def check_users(tenant_id, digest_login, env_conf, config): __abort_script("Okay, not doing anything.") else: # create user account for all tenants - for tenant_id in config.tenant_ids: + for tenant_id in CONFIG.tenant_ids: for account in external_api_accounts[tenant_id]: - response = create_user(account, digest_login, config.tenant_urls[tenant_id]) + create_user(account, tenant_id) else: # create user accounts on the specified tenant for account in external_api_accounts[tenant_id]: - create_user(account, digest_login, config.tenant_urls[tenant_id]) + create_user(account, tenant_id) -def get_roles_as_Json_array(account): +def get_roles_as_json_array(account): roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] return roles -def create_user(account, digest_login, tenant_url): +def create_user(account, tenant_id): """ sends a POST request to the admin UI to create a User :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} - :param digest_login: digest login - :param tenant_url: tenant url + :param tenant_id: String :return: """ + log(f"create user {account['username']}") + + tenant_url = CONFIG.tenant_urls[tenant_id] url = '{}/admin-ng/users/'.format(tenant_url) data = { 'username': account['username'], 'password': account['password'], 'name': account['name'], 'email': account['email'], - 'roles': str(get_roles_as_Json_array(account)) + 'roles': str(get_roles_as_json_array(account)) } + try: - response = post_request(url, digest_login, '/admin-ng/users/', data=data) - print("created user {}".format(account['username'])) + response = post_request(url, DIGEST_LOGIN, '/admin-ng/users/', data=data) except RequestError as err: if err.get_status_code() == "409": print("Conflict, a user with username {} already exist.".format(account['username'])) @@ -71,17 +85,19 @@ def create_user(account, digest_login, tenant_url): return response -def get_user(username, digest_login, tenant_url): +def get_user(username, tenant_id): """ sends a GET request to the admin UI to get a User :param username: String - :param digest_login: digest login - :param tenant_url: tenant url + :param tenant_id: String :return: """ + + tenant_url = CONFIG.tenant_urls[tenant_id] url = f'{tenant_url}/admin-ng/users/{username}.json' + try: - response = get_request(url, digest_login, '/admin-ng/users/{username}.json') + response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') except RequestError as err: if err.get_status_code() == "404": return False From c7abf8044f074f31f5c99d88593c36e86bbe1e35 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 14:19:37 +0200 Subject: [PATCH 21/79] Fixed function call get_user with the new parameters --- multi-tenant-configuration/configure_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index d7985fe..79ea926 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -127,7 +127,7 @@ def check_group_members(group, existing_group, tenant_id): members = existing_group_members.copy() missing_members = [member for member in group_members if member not in existing_group_members] for member in missing_members: - if not get_user(username=member, digest_login=DIGEST_LOGIN, tenant_url=tenant_url): + if not get_user(username=member, tenant_id=tenant_id): log(f"Member {member} of group {group['name']} not found on tenant {tenant_id}.") missing_members.remove(member) additional_members = [member for member in existing_group_members if member not in group_members] @@ -361,7 +361,7 @@ def create_group(group, tenant_id): members = extract_members_from_group(group, tenant_id) # check if member exist on tenant for member in members: - if not get_user(username=member, digest_login=DIGEST_LOGIN, tenant_url=tenant_url): + if not get_user(username=member, tenant_id=tenant_id): print(f"Member {member} does not exist.") members.remove(member) members = ",".join(members) From 078bea02e498286cd98400d02cee700f4caef982 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 14:52:45 +0200 Subject: [PATCH 22/79] Improved args parsing It is now checked if the argument 'check' has a valid value. --- .../parsing_configurations.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index a62ef2c..d64e030 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -2,13 +2,14 @@ import json from args.args_parser import get_args_parser from args.args_error import args_error -from rest_requests.request import get_request, post_request -from rest_requests.request_error import RequestError +# from rest_requests.request import get_request, post_request +# from rest_requests.request_error import RequestError from rest_requests.basic_requests import get_tenants VERBOSE_FLAG = True + def parse_args(): """ Parse the arguments and check them for correctness @@ -35,8 +36,14 @@ def parse_args(): if len(args.environment) > 1: args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") - if not args.tenantid: args.tenantid = [''] - if not args.check: args.check = ['all'] + if not args.tenantid: + args.tenantid = [''] + + if not args.check: + args.check = ['all'] + elif args.check[0] not in ['users', 'groups', 'cast', 'capture']: + args_error(parser, "The check should be 'users', 'groups', 'cast' or 'capture'") + global VERBOSE_FLAG if args.verbose and args.verbose[0] == "True": VERBOSE_FLAG = True From af8ecbe5f7941842e04375b21ad7d87ba1389152 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 9 Jun 2021 22:06:40 +0200 Subject: [PATCH 23/79] Added improved IO to User checks and restructured code The script can now store permissions for creating accounts if the user wants this. imrpoved readability and structure of the code. The basic structure is now similar to the group checks. More checks need to be added. --- multi-tenant-configuration/configure_users.py | 81 ++++++++++++++----- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index a6a1c91..d63be28 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -4,7 +4,8 @@ # from args.args_error import args_error from rest_requests.request import get_request, post_request from rest_requests.request_error import RequestError -from input_output.input import get_yes_no_answer +# from input_output.input import get_yes_no_answer +from user_interaction import check_or_ask_for_permission from parsing_configurations import __abort_script, log @@ -23,26 +24,42 @@ def check_users(tenant_id, digest_login, env_conf, config): ENV_CONFIG = env_conf CONFIG = config - external_api_accounts = {} - for tenant in ENV_CONFIG['opencast_organizations']: - id = tenant['id'] - # ToDo check if this is necessary - if id != "dummy": - external_api_accounts[id] = tenant['external_api_accounts'] - - if not tenant_id: - for_all_tenants = get_yes_no_answer("Create User for all tenants?") - if not for_all_tenants: - __abort_script("Okay, not doing anything.") - else: - # create user account for all tenants - for tenant_id in CONFIG.tenant_ids: - for account in external_api_accounts[tenant_id]: - create_user(account, tenant_id) + # Check and configure System User Accounts & External API User Accounts: + # For all organizations: + for organization in ENV_CONFIG['opencast_organizations']: + if organization['id'] == tenant_id: # ToDo or 'all' ? + for user in organization['external_api_accounts']: + # For all Users # ToDo System & External API ? + # check and configure user + check_user(user=user, tenant_id=tenant_id) + + +def check_user(user, tenant_id): + log(f"Check user {user['name']} on tenant {tenant_id}.") + + # Check if user exists + existing_user = get_user(username=user['username'], tenant_id=tenant_id) + if not existing_user: + # create user if it does not exist on tenant (Ask for permission) + action_allowed = check_or_ask_for_permission( + target_type='user', + action='create', + target_name=user['name'], + tenant_id=tenant_id + ) + if action_allowed: + create_user(account=user, tenant_id=tenant_id) else: - # create user accounts on the specified tenant - for account in external_api_accounts[tenant_id]: - create_user(account, tenant_id) + print('User already exist.') + # ToDo checks + + # Check if Account has External API access. (/api/info/me & /api/info/me/roles) + check_api_access(user=user, tenant_id=tenant_id) + + # Check if Roles (from API request?) match roles in the configuration file. + + # If no External API access or roles do not match: + # Update account (Asks for permission) def get_roles_as_json_array(account): @@ -58,7 +75,7 @@ def create_user(account, tenant_id): :param tenant_id: String :return: """ - log(f"create user {account['username']}") + log(f"Create user {account['username']}") tenant_url = CONFIG.tenant_urls[tenant_id] url = '{}/admin-ng/users/'.format(tenant_url) @@ -75,7 +92,7 @@ def create_user(account, tenant_id): except RequestError as err: if err.get_status_code() == "409": print("Conflict, a user with username {} already exist.".format(account['username'])) - if err.get_status_code() == "403": + elif err.get_status_code() == "403": print("Forbidden, not enough permissions to create a user with a admin role.") return False except Exception as e: @@ -85,6 +102,26 @@ def create_user(account, tenant_id): return response +def check_api_access(user, tenant_id): + # ToDo + + tenant_url = CONFIG.tenant_urls[tenant_id] + url = '{}/api/info/me/'.format(tenant_url) + data = { + 'tenant_id': tenant_id, + 'username': user['username'], + 'password': user['password'] + } + + try: + response = get_request(url, DIGEST_LOGIN, '/api/info/me', data=data) + print(response.json()) + except Exception as e: + print(e) + + return + + def get_user(username, tenant_id): """ sends a GET request to the admin UI to get a User From 9f2e64c7a22665bc265e4aabb5f25bc7b5073347 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 21 Jun 2021 20:10:15 +0200 Subject: [PATCH 24/79] Added header argument to get_request in request.py One can now add individual headers to a get request. --- lib/rest_requests/request.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/rest_requests/request.py b/lib/rest_requests/request.py index 8b06131..dedd182 100644 --- a/lib/rest_requests/request.py +++ b/lib/rest_requests/request.py @@ -11,7 +11,7 @@ def get_request(url, digest_login, element_description, asset_type_description=None, asset_description=None, - stream=False): + stream=False, headers=None): """ Make a get request to the given url with the given digest login. If the request fails with an error or a status code != 200, a Request Error with the error message /status code and the given descriptions is thrown. @@ -28,13 +28,18 @@ def get_request(url, digest_login, element_description, asset_type_description=N :type asset_description: str :param stream: Whether to stream response :type stream: bool + :param headers: The headers to include in the request + :type headers: dict :return: response :raise RequestError: """ + auth = HTTPDigestAuth(digest_login.user, digest_login.password) + headers = headers if headers else {} + headers["X-Requested-Auth"] = "Digest" + try: - response = requests.get(url, auth=HTTPDigestAuth(digest_login.user, digest_login.password), - headers={"X-Requested-Auth": "Digest"}, stream=stream) + response = requests.get(url, auth=auth, headers=headers, stream=stream) except Exception as e: raise RequestError.with_error(url, str(e), element_description, asset_type_description, asset_description) From ef2e9ece3bbec86c036ab8552864863b37dc5b52 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 21 Jun 2021 20:12:47 +0200 Subject: [PATCH 25/79] Updated the way permissions are stored Permissions are now stored in a dictionary via a combined key of 'action' + 'tenant' + 'target' target-specific permissions overwrite tenant-specific permissions. --- .../user_interaction.py | 97 ++++++++++++------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py index d9639c1..19bb5d0 100644 --- a/multi-tenant-configuration/user_interaction.py +++ b/multi-tenant-configuration/user_interaction.py @@ -1,15 +1,16 @@ from parsing_configurations import log -from collections import namedtuple +# from collections import namedtuple import re -Permission = namedtuple('Permission', ['tenant', 'target', 'permission_value']) +# Permission = namedtuple('Permission', ['tenant', 'target', 'permission_value']) permissions = { 'user': {}, 'group': {} } ANSWER_PATTERN = r"^[yn]$|^[yn][ta][ta]$" HELP_OPTION = 'h' +ALL = 'all' def check_or_ask_for_permission(target_type, action, target_name, tenant_id, option_i=False) -> bool: @@ -29,37 +30,52 @@ def get_permission(target_type, action, target_name, tenant_id) -> bool: log('permissions: ', permissions) - permission = None - target_permission = None - tenant_permission = None - - try: - for p in permissions[target_type][action]: - # most specific permission - if p.tenant == tenant_id and p.target == target_name: - permission = p.permission_value - break - # either tenant or target specific - elif p.tenant == 'all' and p.target == target_name: - target_permission = p.permission_value - elif p.tenant == tenant_id and p.target == 'all': - tenant_permission = p.permission_value - # most general permission - elif p.tenant == 'all' and p.target == 'all': - permission = p.permission_value - - # target permission is prioritized over tenant permission - # both will overwrite a general permission - if target_permission is not None: - permission = target_permission - elif tenant_permission is not None: - permission = tenant_permission + # permission = None + # target_permission = None + # tenant_permission = None + + key = __build_key(action, tenant_id, target_name) + if key in permissions[target_type].keys(): + return permissions[target_type][key] + key = __build_key(action, tenant='all', target=target_name) + if key in permissions[target_type].keys(): + return permissions[target_type][key] + key = __build_key(action, tenant=tenant_id, target='all') + if key in permissions[target_type].keys(): + return permissions[target_type][key] + key = __build_key(action, tenant='all', target='all') + if key in permissions[target_type].keys(): + return permissions[target_type][key] + + return None + + # try: + # for p in permissions[target_type][action]: + # # most specific permission + # if p.tenant == tenant_id and p.target == target_name: + # permission = p.permission_value + # break + # # either tenant or target specific + # elif p.tenant == 'all' and p.target == target_name: + # target_permission = p.permission_value + # elif p.tenant == tenant_id and p.target == 'all': + # tenant_permission = p.permission_value + # # most general permission + # elif p.tenant == 'all' and p.target == 'all': + # permission = p.permission_value + # + # # target permission is prioritized over tenant permission + # # both will overwrite a general permission + # if target_permission is not None: + # permission = target_permission + # elif tenant_permission is not None: + # permission = tenant_permission # if no permission is found, None is returned - except KeyError: - print('no permission found') - - return permission + # except KeyError: + # print('no permission found') + # + # return permission def ask_user(target_type, action, target_name, tenant_id, option_i=False) -> str: @@ -98,6 +114,10 @@ def parsable(answer) -> bool: return False +def __build_key(action, tenant, target): + return action + ':' + tenant + ':' + target + + def process_answer(answer, target_type, action, target_name, tenant_id, option_i) -> bool: # simple yes or no case (not stored) @@ -113,12 +133,15 @@ def process_answer(answer, target_type, action, target_name, tenant_id, option_i permission_value = True if answer.startswith('y') else False tenant = 'all' if answer[1] == 'a' else tenant_id target = 'all' if answer[2] == 'a' else target_name - p = Permission(tenant, target, permission_value) - - try: - permissions[target_type][action].append(p) - except: - permissions[target_type][action] = [p] + # p = Permission(tenant, target, permission_value) + key = __build_key(action, tenant, target) + permissions[target_type][key] = permission_value + + # try: + # # permissions[target_type][action].append(p) + # permissions[target_type][action][tenant][target] = permission_value + # except: + # permissions[target_type][action] = [p] return permission_value From f66ba67e44136ea83ced6cd42a132b60b5383184 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 21 Jun 2021 20:15:32 +0200 Subject: [PATCH 26/79] Added checks to configure_users.py The script now checks the API acess and the user roles. A new function to update a user is also added. --- multi-tenant-configuration/configure_users.py | 171 ++++++++++++++++-- 1 file changed, 153 insertions(+), 18 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index d63be28..a477e4c 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -2,9 +2,9 @@ # import json # from args.args_parser import get_args_parser # from args.args_error import args_error -from rest_requests.request import get_request, post_request +from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError -# from input_output.input import get_yes_no_answer +from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission from parsing_configurations import __abort_script, log @@ -51,19 +51,20 @@ def check_user(user, tenant_id): create_user(account=user, tenant_id=tenant_id) else: print('User already exist.') - # ToDo checks - # Check if Account has External API access. (/api/info/me & /api/info/me/roles) + # Check if Account has External API access. (/api/info/me) check_api_access(user=user, tenant_id=tenant_id) - - # Check if Roles (from API request?) match roles in the configuration file. - + # Check if Roles match roles in the configuration file. (/api/info/me/roles) + check_user_roles(user=user, tenant_id=tenant_id) # If no External API access or roles do not match: # Update account (Asks for permission) -def get_roles_as_json_array(account): +def __get_roles_as_json_array(account, as_string=False): roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + if as_string: + roles = [str(role) for role in roles] + roles = '[' + ','.join(roles) + ']' return roles @@ -84,7 +85,7 @@ def create_user(account, tenant_id): 'password': account['password'], 'name': account['name'], 'email': account['email'], - 'roles': str(get_roles_as_json_array(account)) + 'roles': __get_roles_as_json_array(account, as_string=True) } try: @@ -102,24 +103,158 @@ def create_user(account, tenant_id): return response -def check_api_access(user, tenant_id): - # ToDo +def update_user(tenant_id, user_id=None, user=None, name=None, email=None, roles=None): + log(f"Try to update user ... ") + + if not user_id and not user: + log("Cannot update user without a specified name.") + return False + + if user: + if not user_id: + user_id = user['username'] + if not name: + name = user['name'] + if not email: + email = user['email'] + if not roles: + roles = user['roles'] + + if not isinstance(roles, list): + roles = [roles] + # roles = [str({'name': role, 'type': 'INTERNAL'}) for role in roles] + roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) + + print(roles) tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/api/info/me/'.format(tenant_url) + url = f'{tenant_url}/admin-ng/users/{user_id}.json' + data = { - 'tenant_id': tenant_id, - 'username': user['username'], - 'password': user['password'] + 'name': name, + 'email': email, + 'roles': roles } try: - response = get_request(url, DIGEST_LOGIN, '/api/info/me', data=data) - print(response.json()) + response = put_request(url, DIGEST_LOGIN, '/api/groups/{username}.json', data=data) + except RequestError as err: + if err.get_status_code() == "400": # ToDo: check if this is actually 404 + print(f"Bad Request: Invalid data provided.") + print("RequestError: ", err) + return False + except Exception as e: + print(f"User with name {name} could not be updated. \n", "Exception: ", str(e)) + return False + + log(f"Updated user {name}.") + + return response + + +def check_api_access(user, tenant_id): + log(f"Check API access for user {user['username']}") + + if not get_user_info(user=user, tenant_id=tenant_id): + # ToDo ask for permission to solve the problem + print('User has no API Access') + + +def check_user_roles(user, tenant_id): + log(f"Check user roles of user {user['username']}") + + existing_user_roles = get_user_roles(user, tenant_id) + user_roles = user['roles'] + + print('system roles: ', existing_user_roles) + print('config roles: ', user_roles) + + roles = existing_user_roles.copy() + missing_roles = [role for role in user_roles if role not in existing_user_roles] + additional_roles = [role for role in existing_user_roles if role not in user_roles] + + if user_roles == existing_user_roles: + log('User roles match.') + else: + if missing_roles: + print("Missing roles: ", missing_roles) + action_allowed = check_or_ask_for_permission( + target_type='user', + action='add missing user roles', + target_name=user['name'], + tenant_id=tenant_id, + option_i=True + ) + if action_allowed == 'i': + for role in missing_roles: + action_allowed = get_yes_no_answer(f"Add role {role} to user {user['name']}?") + if action_allowed: + roles.append(role) + elif action_allowed: + for role in missing_roles: + roles.append(role) + + if additional_roles: + print("Additional roles: ", additional_roles) + action_allowed = check_or_ask_for_permission( + target_type='user', + action='remove additional user roles', + target_name=user['name'], + tenant_id=tenant_id, + option_i=True + ) + if action_allowed == 'i': + for role in additional_roles: + action_allowed = get_yes_no_answer(f"Remove role {role} from user {user['name']}?") + if action_allowed: + roles.remove(role) + elif action_allowed: + for role in additional_roles: + roles.remove(role) + + if roles != existing_user_roles: + # roles = ",".join(roles) + print(roles) + update_user(tenant_id=tenant_id, user=user, roles=roles) + + return + + +def get_user_info(user, tenant_id): + + tenant_url = CONFIG.tenant_urls[tenant_id] + url = '{}/api/info/me'.format(tenant_url) + headers = { + 'X-RUN-AS-USER': user['username'] + } + + try: + response = get_request(url, DIGEST_LOGIN, '/api/info/me', headers=headers) + except Exception as e: + print(e) + return False + + return response.json() + + +def get_user_roles(user, tenant_id): + # ToDo check if the 'effective roles' should be excluded here + # -> switch to /admin-ng/users/{username}.json + + tenant_url = CONFIG.tenant_urls[tenant_id] + url = '{}/api/info/me/roles'.format(tenant_url) + headers = { + 'X-RUN-AS-USER': user['username'] + } + + try: + response = get_request(url, DIGEST_LOGIN, '/api/info/me/roles', headers=headers) except Exception as e: print(e) + return False + + return response.json() - return def get_user(username, tenant_id): From 19bddc68bfcc719c94e3bc2f56c6ee4091632599 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 23 Jun 2021 10:59:22 +0200 Subject: [PATCH 27/79] minor code cleanup --- multi-tenant-configuration/configure_groups.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 79ea926..a78d06a 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -5,7 +5,7 @@ from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError from configure_users import get_user -from input_output.input import get_yes_no_answer, get_configurable_answer +from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission from parsing_configurations import log @@ -310,6 +310,7 @@ def group_description_template(description, tenant_id): def update_group(tenant_id, group=None, name=None, description=None, roles=None, members=None): log(f"Try to update group ... ") + if not name and not group: log("Cannot update group without a specified name.") return False @@ -335,7 +336,7 @@ def update_group(tenant_id, group=None, name=None, description=None, roles=None, 'roles': roles, 'members': members, } - print('data ', data) + try: response = put_request(url, DIGEST_LOGIN, '/api/groups/{groupId}', data=data) except RequestError as err: @@ -348,6 +349,7 @@ def update_group(tenant_id, group=None, name=None, description=None, roles=None, return False log(f"Updated group {name}.") + return response From 3a8f52e7bcb492c0118dcc999f19be3b3cb6d0e6 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 12 Jul 2021 13:35:08 +0200 Subject: [PATCH 28/79] removed unused imports --- multi-tenant-configuration/main.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 1433091..7f185bf 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -2,18 +2,10 @@ import sys sys.path.append(os.path.join(os.path.abspath('..'), "lib")) -# import io -# import yaml -# from args.args_parser import get_args_parser -# from args.args_error import args_error -# from rest_requests.request_error import RequestError -# from input_output.input import get_yes_no_answer from args.digest_login import DigestLogin from parsing_configurations import parse_args, read_yaml_file, parse_config from configure_users import check_users from configure_groups import check_groups -# from rest_requests.request import get_request, post_request -# from rest_requests.request_error import RequestError import config From d2bab49c8ba3f08d9ab8332201eb3e8cab78de74 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 12 Jul 2021 13:40:12 +0200 Subject: [PATCH 29/79] removed more unused imports --- multi-tenant-configuration/configure_groups.py | 4 ---- multi-tenant-configuration/configure_users.py | 4 ---- multi-tenant-configuration/parsing_configurations.py | 2 -- multi-tenant-configuration/user_interaction.py | 2 -- 4 files changed, 12 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index a78d06a..6e0b337 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -1,7 +1,3 @@ -# import yaml -# import json -# from args.args_parser import get_args_parser -# from args.args_error import args_error from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError from configure_users import get_user diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index a477e4c..8c0c6b5 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -1,7 +1,3 @@ -# import yaml -# import json -# from args.args_parser import get_args_parser -# from args.args_error import args_error from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError from input_output.input import get_yes_no_answer diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index d64e030..7cf866c 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -2,8 +2,6 @@ import json from args.args_parser import get_args_parser from args.args_error import args_error -# from rest_requests.request import get_request, post_request -# from rest_requests.request_error import RequestError from rest_requests.basic_requests import get_tenants diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py index 19bb5d0..2b01790 100644 --- a/multi-tenant-configuration/user_interaction.py +++ b/multi-tenant-configuration/user_interaction.py @@ -1,9 +1,7 @@ from parsing_configurations import log -# from collections import namedtuple import re -# Permission = namedtuple('Permission', ['tenant', 'target', 'permission_value']) permissions = { 'user': {}, 'group': {} From ba009f573308517a48e641660e893e8ef3e573ba Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 12 Jul 2021 14:35:18 +0200 Subject: [PATCH 30/79] updated README --- multi-tenant-configuration/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index d9543a8..4d2a08e 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -35,27 +35,27 @@ access to the events/series) and the same password to both tenants._ The names in the group config file must be unique per Tenant! ### Usage -**ToDo** -The script can be called with the following parameters (all parameters in brackets are optional): +The script can be called with the following command (all parameters in brackets are optional): -`main.py ... ` +`python main.py -e ENVIRONMENT [-t TENANT_ID] [-c CHECK] [-v True]` | Param | Description | | :---: | :---------- | -| `-t` / `--tenant` | The id(s) of the tenant to be configured | | `-e` / `--environment` | The environment where to find the configuration file (either `staging` or `production`) | -| ... / ... | ... | +| `-t` / `--tenantid` | The id of the target tenant to be configured | +| `-c` / `--check` | checks to be performed (`users`, `groups`, `cast` or `capture`) (default: `all`) | +| `-v` / `--verbose` | enables logging to be prompted if set to `True` | -#### Usage example -**ToDo** +#### example: -`main.py ... ` +`python main.py -e staging -t tenant1 -c groups -v True` ## Requirements -**ToDo** -This scrypt was written for Python 3.8. You can install the necessary packages with +This script was written for Python 3.8. You can install the necessary packages with + +**ToDo check the requirements file** `pip install -r requirements.txt` From 69db473717e7f7a89f90aa1a8f9feebec7e6fb96 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 12 Jul 2021 15:00:52 +0200 Subject: [PATCH 31/79] code cleanup --- .../parsing_configurations.py | 7 +- .../user_interaction.py | 65 ++++--------------- 2 files changed, 14 insertions(+), 58 deletions(-) diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 7cf866c..795f3d8 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -17,13 +17,12 @@ def parse_args(): """ parser, optional_args, required_args = get_args_parser() - # ToDo change optional to required_args ? required_args.add_argument("-e", "--environment", type=str, nargs='+', help="the environment (either 'staging' or 'production')") optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") optional_args.add_argument("-c", "--check", type=str, nargs='+', help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") - optional_args.add_argument("-v", "--verbose", type=str, nargs='+',help="enables more logging") + optional_args.add_argument("-v", "--verbose", type=str, nargs='+', help="enables more logging") args = parser.parse_args() @@ -90,9 +89,9 @@ def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test """ with open(json_file_path, 'r') as json_file: - jsonData = json.load(json_file) + json_data = json.load(json_file) with open(yaml_file_path, 'w') as file: - yaml.dump(jsonData, file, sort_keys=False) + yaml.dump(json_data, file, sort_keys=False) return True diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py index 2b01790..1027413 100644 --- a/multi-tenant-configuration/user_interaction.py +++ b/multi-tenant-configuration/user_interaction.py @@ -8,7 +8,6 @@ } ANSWER_PATTERN = r"^[yn]$|^[yn][ta][ta]$" HELP_OPTION = 'h' -ALL = 'all' def check_or_ask_for_permission(target_type, action, target_name, tenant_id, option_i=False) -> bool: @@ -16,9 +15,9 @@ def check_or_ask_for_permission(target_type, action, target_name, tenant_id, opt # check if permission is already defined permission = get_permission(target_type, action, target_name, tenant_id) if permission is None: - # otherwise ask for user input + # otherwise ask for user input ... answer = ask_user(target_type, action, target_name, tenant_id, option_i) - # process answer and update permissions + # ... and process answer and update permissions permission = process_answer(answer, target_type, action, target_name, tenant_id, option_i) return permission @@ -28,10 +27,6 @@ def get_permission(target_type, action, target_name, tenant_id) -> bool: log('permissions: ', permissions) - # permission = None - # target_permission = None - # tenant_permission = None - key = __build_key(action, tenant_id, target_name) if key in permissions[target_type].keys(): return permissions[target_type][key] @@ -47,61 +42,31 @@ def get_permission(target_type, action, target_name, tenant_id) -> bool: return None - # try: - # for p in permissions[target_type][action]: - # # most specific permission - # if p.tenant == tenant_id and p.target == target_name: - # permission = p.permission_value - # break - # # either tenant or target specific - # elif p.tenant == 'all' and p.target == target_name: - # target_permission = p.permission_value - # elif p.tenant == tenant_id and p.target == 'all': - # tenant_permission = p.permission_value - # # most general permission - # elif p.tenant == 'all' and p.target == 'all': - # permission = p.permission_value - # - # # target permission is prioritized over tenant permission - # # both will overwrite a general permission - # if target_permission is not None: - # permission = target_permission - # elif tenant_permission is not None: - # permission = tenant_permission - - # if no permission is found, None is returned - # except KeyError: - # print('no permission found') - # - # return permission - def ask_user(target_type, action, target_name, tenant_id, option_i=False) -> str: - help_description = "Valid answers are: \nHELP DESCRIPTION" - individual_option = "\n Write 'i' to perform the action individually for each case. " if option_i else "" - question = f"""Do you want to {action} ({target_type} {target_name} on {tenant_id})? - Write 'y' to perform the action. Write 'n' to skipp this action. {individual_option}Write '{HELP_OPTION}' for help. + help_description = f""" Write 'y' to perform the action. Write 'n' to skipp this action. {individual_option} Add 't' for 'tenant' or 'a' for 'all' to store your decision for this or all tenants. Add 't' for 'target' or 'a' for 'all' to store your decision for this or all targets. EXAMPLE: Write 'yat' to store your decision for the action on ALL tenants and for THIS target. """ - answer = '' + question = f"Do you want to {action} ({target_type} {target_name} on {tenant_id})? Write '{HELP_OPTION}' for help.\n" + + # ask the question + answer = input(question).lower() while True: # catch the help option: give a more detailed description of the options if answer == HELP_OPTION: answer = input(help_description) - # ask the question - else: - answer = input(question).lower() # return all valid answers - if parsable(answer) or (option_i and answer == 'i'): + elif parsable(answer) or (option_i and answer == 'i'): return answer + # catch all invalid answers else: - print("Invalid answer.\n") + answer = input(f"Invalid answer. Write '{HELP_OPTION}' for help.\n").lower() def parsable(answer) -> bool: @@ -125,21 +90,13 @@ def process_answer(answer, target_type, action, target_name, tenant_id, option_i return False # individual case if option_i and answer == 'i': - return answer + return 'i' # store answer if user specified this permission_value = True if answer.startswith('y') else False tenant = 'all' if answer[1] == 'a' else tenant_id target = 'all' if answer[2] == 'a' else target_name - # p = Permission(tenant, target, permission_value) key = __build_key(action, tenant, target) permissions[target_type][key] = permission_value - # try: - # # permissions[target_type][action].append(p) - # permissions[target_type][action][tenant][target] = permission_value - # except: - # permissions[target_type][action] = [p] - return permission_value - From cd17f0ec759e85fe2365428ed69af9eb5bf4e41f Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 12 Jul 2021 16:22:12 +0200 Subject: [PATCH 32/79] Improved config parsing and code cleanup The tenant URLs can now be set for specific tenants. For tenants where no URL is set, the URL pattern will be used. The config is now set once in the beginning of the script and not for every check. --- multi-tenant-configuration/config.py | 8 ++--- .../configure_groups.py | 32 ++++++++----------- multi-tenant-configuration/configure_users.py | 31 ++++++++---------- multi-tenant-configuration/main.py | 14 ++++---- .../parsing_configurations.py | 7 ++-- 5 files changed, 44 insertions(+), 48 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 93e2330..92b8b81 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -3,17 +3,17 @@ # Set this to your global admin node base_url = "http://localhost:8080" -# If you have multiple tenants use an URL pattern. # ToDo otherwise, this can be empty or commented out +# If you have multiple tenants use an URL pattern. # example: tenant_url_pattern = "https://{}.example.org" tenant_url_pattern = "http://{}:8080" -# ToDo You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: -# # example: -# tenant_urls = { '': 'http://tenant1:8080', '': 'http://tenant2:8080' } +# You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: +# # examples: # tenant_urls = { # 'tenant1': 'http://tenant1:8080', # 'tenant2': 'http://tenant2:8080' # } +# tenant_urls = {'tenant1': 'https://develop.opencast.org'} # digest user digest_user = "opencast_system_account" diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 6e0b337..fdf1d9c 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -11,8 +11,7 @@ DIGEST_LOGIN = None -def check_groups(tenant_id, digest_login, group_config, config): - log('\nStart checking groups for tenant: ', tenant_id) +def set_config_groups(digest_login, group_config, config): global DIGEST_LOGIN global GROUP_CONFIG @@ -20,10 +19,15 @@ def check_groups(tenant_id, digest_login, group_config, config): DIGEST_LOGIN = digest_login GROUP_CONFIG = group_config CONFIG = config - # tenant_url = CONFIG.tenant_urls[tenant_id] + + return + + +def check_groups(tenant_id): + log('\nStart checking groups for tenant: ', tenant_id) # For all Groups: - for group in group_config['groups']: + for group in GROUP_CONFIG['groups']: # Check group if group['tenants'] == 'all' or group['tenants'] == tenant_id: group['identifier'] = generate_group_identifier(group, tenant_id) @@ -51,7 +55,6 @@ def check_group(group, tenant_id): # Update them if they do not match. (Asks for permission) check_group_description(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. - # ToDo Create missing group members. (Asks for permission) ? # Check if group members match the group members provided in the configuration. # Add or remove members accordingly. check_group_members(group=group, existing_group=existing_group, tenant_id=tenant_id) @@ -67,8 +70,7 @@ def check_group(group, tenant_id): def check_if_group_exists(group, tenant_id): log(f"check if group {group['name']} exists.") - tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/api/groups/{}'.format(tenant_url, group['identifier']) + url = f"{CONFIG.tenant_urls[tenant_id]}/api/groups/{group['identifier']}" try: response = get_request(url, DIGEST_LOGIN, '/api/groups/') return response.json() @@ -78,7 +80,7 @@ def check_if_group_exists(group, tenant_id): else: raise Exception except Exception as e: - print("ERROR: {}".format(str(e))) + print("ERROR: ", str(e)) return False @@ -111,9 +113,6 @@ def check_group_description(group, existing_group, tenant_id): def check_group_members(group, existing_group, tenant_id): log(f"Check members for group {group['name']}.") - # ToDo remove this also from configure_users - tenant_url = CONFIG.tenant_urls[tenant_id] - group_members = extract_members_from_group(group=group, tenant_id=tenant_id) existing_group_members = sorted(filter(None, existing_group['members'].split(","))) @@ -244,15 +243,14 @@ def generate_group_identifier(group, tenant_id): def get_groups_from_tenant(tenant_id): - tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/api/groups/'.format(tenant_url) + url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' try: response = get_request(url, DIGEST_LOGIN, '/api/groups/') except RequestError as err: print('RequestError: ', err) return False except Exception as e: - print(f"Groups could not be retrieved from {tenant_url}. \n", "Error: ", str(e)) + print("Groups could not be retrieved. \n", "Error: ", str(e)) return False return response.json() @@ -323,8 +321,7 @@ def update_group(tenant_id, group=None, name=None, description=None, roles=None, description = group_description_template(group['description'], tenant_id) else: group_id = generate_group_identifier(group={'name': name}, tenant_id=tenant_id) - tenant_url = CONFIG.tenant_urls[tenant_id] - url = f'{tenant_url}/api/groups/{group_id}' + url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/{group_id}' data = { 'name': name, @@ -352,8 +349,7 @@ def update_group(tenant_id, group=None, name=None, description=None, roles=None, def create_group(group, tenant_id): log(f"trying to create group {group['name']}. ") - tenant_url = CONFIG.tenant_urls[tenant_id] - url = f'{tenant_url}/api/groups/' + url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' # extract members and roles members = extract_members_from_group(group, tenant_id) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 8c0c6b5..ad25fc9 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -10,8 +10,7 @@ DIGEST_LOGIN = None -def check_users(tenant_id, digest_login, env_conf, config): - log('\nStart checking users for tenant: ', tenant_id) +def set_config_users(digest_login, env_conf, config): global DIGEST_LOGIN global ENV_CONFIG @@ -20,6 +19,12 @@ def check_users(tenant_id, digest_login, env_conf, config): ENV_CONFIG = env_conf CONFIG = config + return + + +def check_users(tenant_id): + log('\nStart checking users for tenant: ', tenant_id) + # Check and configure System User Accounts & External API User Accounts: # For all organizations: for organization in ENV_CONFIG['opencast_organizations']: @@ -46,8 +51,6 @@ def check_user(user, tenant_id): if action_allowed: create_user(account=user, tenant_id=tenant_id) else: - print('User already exist.') - # Check if Account has External API access. (/api/info/me) check_api_access(user=user, tenant_id=tenant_id) # Check if Roles match roles in the configuration file. (/api/info/me/roles) @@ -74,8 +77,7 @@ def create_user(account, tenant_id): """ log(f"Create user {account['username']}") - tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/admin-ng/users/'.format(tenant_url) + url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/' data = { 'username': account['username'], 'password': account['password'], @@ -88,12 +90,12 @@ def create_user(account, tenant_id): response = post_request(url, DIGEST_LOGIN, '/admin-ng/users/', data=data) except RequestError as err: if err.get_status_code() == "409": - print("Conflict, a user with username {} already exist.".format(account['username'])) + print(f"Conflict, a user with username {account['username']} already exist.") elif err.get_status_code() == "403": print("Forbidden, not enough permissions to create a user with a admin role.") return False except Exception as e: - print("User could not be created: {}".format(str(e))) + print("User could not be created: ", str(e)) return False return response @@ -123,9 +125,7 @@ def update_user(tenant_id, user_id=None, user=None, name=None, email=None, roles print(roles) - tenant_url = CONFIG.tenant_urls[tenant_id] - url = f'{tenant_url}/admin-ng/users/{user_id}.json' - + url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user_id}.json' data = { 'name': name, 'email': email, @@ -218,8 +218,7 @@ def check_user_roles(user, tenant_id): def get_user_info(user, tenant_id): - tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/api/info/me'.format(tenant_url) + url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' headers = { 'X-RUN-AS-USER': user['username'] } @@ -237,8 +236,7 @@ def get_user_roles(user, tenant_id): # ToDo check if the 'effective roles' should be excluded here # -> switch to /admin-ng/users/{username}.json - tenant_url = CONFIG.tenant_urls[tenant_id] - url = '{}/api/info/me/roles'.format(tenant_url) + url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me/roles' headers = { 'X-RUN-AS-USER': user['username'] } @@ -261,8 +259,7 @@ def get_user(username, tenant_id): :return: """ - tenant_url = CONFIG.tenant_urls[tenant_id] - url = f'{tenant_url}/admin-ng/users/{username}.json' + url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{username}.json' try: response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 7f185bf..3bb3b05 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -4,8 +4,8 @@ from args.digest_login import DigestLogin from parsing_configurations import parse_args, read_yaml_file, parse_config -from configure_users import check_users -from configure_groups import check_groups +from configure_users import check_users, set_config_users +from configure_groups import check_groups, set_config_groups import config @@ -18,6 +18,8 @@ def main(): env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file script_config = parse_config(config, env_conf, digest_login) # parse config.py group_config = read_yaml_file(script_config.group_path) # read group config file + set_config_users(digest_login=digest_login, env_conf=env_conf, config=script_config) + set_config_groups(digest_login=digest_login, group_config=group_config, config=script_config) # if tenant is not given, we perform the checks for all tenants if tenant_id: @@ -28,12 +30,12 @@ def main(): ### Start checks ### for tenant_id in tenants_to_check: if check == 'all': - check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) - check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + check_users(tenant_id=tenant_id) + check_groups(tenant_id=tenant_id) elif check == 'users': - check_users(tenant_id=tenant_id, digest_login=digest_login, env_conf=env_conf, config=script_config) + check_users(tenant_id=tenant_id) elif check == 'groups': - check_groups(tenant_id=tenant_id, digest_login=digest_login, group_config=group_config, config=script_config) + check_groups(tenant_id=tenant_id) if __name__ == '__main__': diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 795f3d8..6a73ba5 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -65,15 +65,16 @@ def read_yaml_file(path): def parse_config(config, env_config, digest_login): - # ToDo Check if all mandatory configurations are given + # ToDo Check if all mandatory configurations are given i.e. url pattern or a url for all tenants # ToDo should mh_default_org be removed from tenant_ids? config.tenant_ids = get_tenants(config.base_url, digest_login) config.tenant_ids.remove('mh_default_org') - if not (hasattr(config, 'tenant_urls') and config.tenant_urls): + if not hasattr(config, 'tenant_urls'): config.tenant_urls = {} - for tenant_id in config.tenant_ids: + for tenant_id in config.tenant_ids: + if not tenant_id in config.tenant_urls: config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) return config From 482917e5c5646461e618404021974d98100c7415 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 14 Jul 2021 12:18:01 +0200 Subject: [PATCH 33/79] minor code cleanup --- multi-tenant-configuration/config.py | 3 +- multi-tenant-configuration/configure_users.py | 32 ++++++------------- multi-tenant-configuration/main.py | 4 +-- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 92b8b81..3e836f0 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -4,7 +4,8 @@ base_url = "http://localhost:8080" # If you have multiple tenants use an URL pattern. -# example: tenant_url_pattern = "https://{}.example.org" +# example: +# tenant_url_pattern = "https://{}.example.org" tenant_url_pattern = "http://{}:8080" # You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index ad25fc9..aed270d 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -26,11 +26,9 @@ def check_users(tenant_id): log('\nStart checking users for tenant: ', tenant_id) # Check and configure System User Accounts & External API User Accounts: - # For all organizations: for organization in ENV_CONFIG['opencast_organizations']: if organization['id'] == tenant_id: # ToDo or 'all' ? - for user in organization['external_api_accounts']: - # For all Users # ToDo System & External API ? + for user in organization['external_api_accounts']: # ToDo System & External API ? # check and configure user check_user(user=user, tenant_id=tenant_id) @@ -51,12 +49,10 @@ def check_user(user, tenant_id): if action_allowed: create_user(account=user, tenant_id=tenant_id) else: - # Check if Account has External API access. (/api/info/me) + # Check if Account has External API access. check_api_access(user=user, tenant_id=tenant_id) - # Check if Roles match roles in the configuration file. (/api/info/me/roles) + # Check if the user roles match the roles in the configuration file. check_user_roles(user=user, tenant_id=tenant_id) - # If no External API access or roles do not match: - # Update account (Asks for permission) def __get_roles_as_json_array(account, as_string=False): @@ -118,26 +114,22 @@ def update_user(tenant_id, user_id=None, user=None, name=None, email=None, roles if not roles: roles = user['roles'] - if not isinstance(roles, list): + if not isinstance(roles, list): # in case only one role is given, make sure roles is a list roles = [roles] - # roles = [str({'name': role, 'type': 'INTERNAL'}) for role in roles] roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) - print(roles) - url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user_id}.json' data = { 'name': name, 'email': email, 'roles': roles } - try: - response = put_request(url, DIGEST_LOGIN, '/api/groups/{username}.json', data=data) + response = put_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json', data=data) except RequestError as err: - if err.get_status_code() == "400": # ToDo: check if this is actually 404 - print(f"Bad Request: Invalid data provided.") print("RequestError: ", err) + if err.get_status_code() == "400": + print(f"Bad Request: Invalid data provided.") return False except Exception as e: print(f"User with name {name} could not be updated. \n", "Exception: ", str(e)) @@ -222,11 +214,10 @@ def get_user_info(user, tenant_id): headers = { 'X-RUN-AS-USER': user['username'] } - try: response = get_request(url, DIGEST_LOGIN, '/api/info/me', headers=headers) except Exception as e: - print(e) + log(e) return False return response.json() @@ -240,7 +231,6 @@ def get_user_roles(user, tenant_id): headers = { 'X-RUN-AS-USER': user['username'] } - try: response = get_request(url, DIGEST_LOGIN, '/api/info/me/roles', headers=headers) except Exception as e: @@ -264,11 +254,9 @@ def get_user(username, tenant_id): try: response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') except RequestError as err: - if err.get_status_code() == "404": - return False - else: + if not err.get_status_code() == "404": print(err) - return False + return False except Exception as e: print(e) return False diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 3bb3b05..d03018f 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -18,8 +18,8 @@ def main(): env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file script_config = parse_config(config, env_conf, digest_login) # parse config.py group_config = read_yaml_file(script_config.group_path) # read group config file - set_config_users(digest_login=digest_login, env_conf=env_conf, config=script_config) - set_config_groups(digest_login=digest_login, group_config=group_config, config=script_config) + set_config_users(digest_login, env_conf, script_config) # import config to the user script + set_config_groups(digest_login, group_config, script_config) # import config to the group script # if tenant is not given, we perform the checks for all tenants if tenant_id: From 78dcbbe1946b6aa20314216ce093422c16c97659 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 14 Jul 2021 21:21:00 +0200 Subject: [PATCH 34/79] get request now allows to also use BasicAuth --- lib/rest_requests/request.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/rest_requests/request.py b/lib/rest_requests/request.py index dedd182..9eb7883 100644 --- a/lib/rest_requests/request.py +++ b/lib/rest_requests/request.py @@ -4,14 +4,14 @@ import os import requests -from requests.auth import HTTPDigestAuth +from requests.auth import HTTPDigestAuth, HTTPBasicAuth from requests_toolbelt import MultipartEncoder from rest_requests.request_error import RequestError -def get_request(url, digest_login, element_description, asset_type_description=None, asset_description=None, - stream=False, headers=None): +def get_request(url, login, element_description, asset_type_description=None, asset_description=None, + stream=False, headers=None, use_digest=True): """ Make a get request to the given url with the given digest login. If the request fails with an error or a status code != 200, a Request Error with the error message /status code and the given descriptions is thrown. @@ -34,9 +34,12 @@ def get_request(url, digest_login, element_description, asset_type_description=N :raise RequestError: """ - auth = HTTPDigestAuth(digest_login.user, digest_login.password) headers = headers if headers else {} - headers["X-Requested-Auth"] = "Digest" + if use_digest: + auth = HTTPDigestAuth(login.user, login.password) + headers["X-Requested-Auth"] = "Digest" + else: + auth = HTTPBasicAuth(login['user'], login['password']) try: response = requests.get(url, auth=auth, headers=headers, stream=stream) From 193f1c68f45f8e7d375478df5619e4d9fca79435 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 14 Jul 2021 21:21:47 +0200 Subject: [PATCH 35/79] added API access check, changed user role checks, and general code cleanup --- multi-tenant-configuration/configure_users.py | 173 +++++++++++------- multi-tenant-configuration/main.py | 11 +- 2 files changed, 117 insertions(+), 67 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index aed270d..a516f23 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -9,6 +9,9 @@ ENV_CONFIG = None DIGEST_LOGIN = None +# ToDo should this be moved to the config file? +UNEXPECTED_ROLES = ["ROLE_ADMIN", "ROLE_ADMIN_UI", "ROLE_UI_", "ROLE_CAPTURE_"] + def set_config_users(digest_login, env_conf, config): @@ -27,14 +30,20 @@ def check_users(tenant_id): # Check and configure System User Accounts & External API User Accounts: for organization in ENV_CONFIG['opencast_organizations']: - if organization['id'] == tenant_id: # ToDo or 'all' ? - for user in organization['external_api_accounts']: # ToDo System & External API ? - # check and configure user - check_user(user=user, tenant_id=tenant_id) + # check switchcast system accounts + if organization['id'] == 'dummy': + log(f'Checking system accounts for tenant {tenant_id} ...') + for system_account in organization['switchcast_system_accounts']: + check_user(system_account, tenant_id) + # check and configure external api accounts + if organization['id'] == tenant_id: # ToDo or 'all' ? + log(f'Checking External API accounts for tenant {tenant_id} ...') + for user in organization['external_api_accounts']: + check_user(user, tenant_id) def check_user(user, tenant_id): - log(f"Check user {user['name']} on tenant {tenant_id}.") + log(f"Checking user {user['name']} on tenant {tenant_id}.") # Check if user exists existing_user = get_user(username=user['username'], tenant_id=tenant_id) @@ -43,16 +52,18 @@ def check_user(user, tenant_id): action_allowed = check_or_ask_for_permission( target_type='user', action='create', - target_name=user['name'], + target_name=user['username'], tenant_id=tenant_id ) if action_allowed: create_user(account=user, tenant_id=tenant_id) else: - # Check if Account has External API access. - check_api_access(user=user, tenant_id=tenant_id) + # Check if password is correct and if the account has External API access. + __check_api_access(user=user, tenant_id=tenant_id) # Check if the user roles match the roles in the configuration file. - check_user_roles(user=user, tenant_id=tenant_id) + __check_user_roles(user, tenant_id) + # check for unexpected roles in the effective roles. + __check_effective_roles(user, tenant_id) def __get_roles_as_json_array(account, as_string=False): @@ -65,8 +76,9 @@ def __get_roles_as_json_array(account, as_string=False): def create_user(account, tenant_id): - """ sends a POST request to the admin UI to create a User - + """ + sends a POST request to the admin UI to create a User + uses the /admin-ng/users/ endpoint :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} :param tenant_id: String :return: @@ -97,29 +109,27 @@ def create_user(account, tenant_id): return response -def update_user(tenant_id, user_id=None, user=None, name=None, email=None, roles=None): - log(f"Try to update user ... ") - - if not user_id and not user: - log("Cannot update user without a specified name.") - return False - - if user: - if not user_id: - user_id = user['username'] - if not name: - name = user['name'] - if not email: - email = user['email'] - if not roles: - roles = user['roles'] - +def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): + log(f"Trying to update user ... ") + + # user_id = user['username'] + # if not name: + # name = user['name'] + # if not email: + # email = user['email'] + # if not roles: + # roles = user['roles'] + name = overwrite_name if overwrite_email else user['name'] + email = overwrite_email if overwrite_email else user['email'] + roles = overwrite_roles if overwrite_roles else user['roles'] + pw = overwrite_pw if overwrite_pw else user['password'] if not isinstance(roles, list): # in case only one role is given, make sure roles is a list roles = [roles] roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) - url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user_id}.json' + url = f"{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user['username']}.json" data = { + 'password': pw, 'name': name, 'email': email, 'roles': roles @@ -140,18 +150,56 @@ def update_user(tenant_id, user_id=None, user=None, name=None, email=None, roles return response -def check_api_access(user, tenant_id): - log(f"Check API access for user {user['username']}") +def __check_api_access(user, tenant_id): + + log(f"Checking API access for user {user['username']}") + + url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' + headers = {} # {'X-RUN-AS-USER': user['username']} + login = { + 'user': user['username'], + 'password': user['password'] + } + + try: + get_request(url, login, '/api/info/me', headers=headers, use_digest=False) + except RequestError: + print(f"User {user['username']} has no API Access") + # ToDo add to group to get API access roles? + action_allowed = check_or_ask_for_permission( + target_type='user', + action='configure user', + target_name=user['username'], + tenant_id=tenant_id + ) + if action_allowed: + update_user(tenant_id, user=user) + except Exception as e: + print('Error: Failed to check for API access.') + print(str(e)) + return False + + return + + +def __check_effective_roles(user, tenant_id): + log(f"Check effective user roles of user {user['username']}") - if not get_user_info(user=user, tenant_id=tenant_id): - # ToDo ask for permission to solve the problem - print('User has no API Access') + effective_user_roles = get_user_roles(user['username'], tenant_id) + for role in effective_user_roles: + for unexpected_role in UNEXPECTED_ROLES: + # ToDo improve this check if role matches unexpected role + if unexpected_role in role: + print(f"Unexpected role found for User {user['username']}: {role}") + + return -def check_user_roles(user, tenant_id): +def __check_user_roles(user, tenant_id): log(f"Check user roles of user {user['username']}") - existing_user_roles = get_user_roles(user, tenant_id) + # ToDo change this to exclude group roles + existing_user_roles = get_user_roles(user['username'], tenant_id) user_roles = user['roles'] print('system roles: ', existing_user_roles) @@ -169,7 +217,7 @@ def check_user_roles(user, tenant_id): action_allowed = check_or_ask_for_permission( target_type='user', action='add missing user roles', - target_name=user['name'], + target_name=user['username'], tenant_id=tenant_id, option_i=True ) @@ -187,7 +235,7 @@ def check_user_roles(user, tenant_id): action_allowed = check_or_ask_for_permission( target_type='user', action='remove additional user roles', - target_name=user['name'], + target_name=user['username'], tenant_id=tenant_id, option_i=True ) @@ -203,34 +251,36 @@ def check_user_roles(user, tenant_id): if roles != existing_user_roles: # roles = ",".join(roles) print(roles) - update_user(tenant_id=tenant_id, user=user, roles=roles) + update_user(tenant_id, user, overwrite_roles=roles) return -def get_user_info(user, tenant_id): +# def get_user_info(user, tenant_id): +# +# url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' +# headers = { +# 'X-RUN-AS-USER': user['username'] +# } +# try: +# response = get_request(url, DIGEST_LOGIN, '/api/info/me', headers=headers) +# except Exception as e: +# log(e) +# return False +# +# return response.json() - url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' - headers = { - 'X-RUN-AS-USER': user['username'] - } - try: - response = get_request(url, DIGEST_LOGIN, '/api/info/me', headers=headers) - except Exception as e: - log(e) - return False - - return response.json() - - -def get_user_roles(user, tenant_id): - # ToDo check if the 'effective roles' should be excluded here - # -> switch to /admin-ng/users/{username}.json +def get_user_roles(user_name, tenant_id): + """ + returns the effective roles of a user (user roles + group roles). + Uses DigestLogin + :param user_name: + :param tenant_id: + :return: + """ url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me/roles' - headers = { - 'X-RUN-AS-USER': user['username'] - } + headers = {'X-RUN-AS-USER': user_name} try: response = get_request(url, DIGEST_LOGIN, '/api/info/me/roles', headers=headers) except Exception as e: @@ -240,17 +290,14 @@ def get_user_roles(user, tenant_id): return response.json() - def get_user(username, tenant_id): - """ sends a GET request to the admin UI to get a User + """ sends a GET request to the admin UI to get a user :param username: String :param tenant_id: String :return: """ - url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{username}.json' - try: response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') except RequestError as err: diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index d03018f..ff2e167 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -30,12 +30,15 @@ def main(): ### Start checks ### for tenant_id in tenants_to_check: if check == 'all': - check_users(tenant_id=tenant_id) - check_groups(tenant_id=tenant_id) + check_users(tenant_id) + check_groups(tenant_id) + # ToDo switchcast_system_accounts(tenant_id) elif check == 'users': - check_users(tenant_id=tenant_id) + check_users(tenant_id) elif check == 'groups': - check_groups(tenant_id=tenant_id) + check_groups(tenant_id) + # elif check == 'capture': + # switchcast_system_accounts(tenant_id) if __name__ == '__main__': From c2520985ffe1ea06aae25dfd892945b0e871be14 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sat, 17 Jul 2021 20:23:38 +0200 Subject: [PATCH 36/79] Added new Checks and more documentation User Checks now contain: - API access (password and API roles check) - User specific roles match config file - Unexpected Group roles --- multi-tenant-configuration/configure_users.py | 151 ++++++++++++------ 1 file changed, 98 insertions(+), 53 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index a516f23..abdade6 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -61,9 +61,9 @@ def check_user(user, tenant_id): # Check if password is correct and if the account has External API access. __check_api_access(user=user, tenant_id=tenant_id) # Check if the user roles match the roles in the configuration file. - __check_user_roles(user, tenant_id) + __check_user_roles(user, existing_user, tenant_id) # check for unexpected roles in the effective roles. - __check_effective_roles(user, tenant_id) + __check_effective_user_roles(user, tenant_id) def __get_roles_as_json_array(account, as_string=False): @@ -79,9 +79,11 @@ def create_user(account, tenant_id): """ sends a POST request to the admin UI to create a User uses the /admin-ng/users/ endpoint - :param account: dict user account to be created (e.g. {'username': 'Peter', 'password': '123'} - :param tenant_id: String - :return: + :param account: The user account to be created (e.g. {'username': 'Peter', 'password': '123'} + :type account: dict + :param tenant_id: The target tenant + :type tenant_id: String + :return: response """ log(f"Create user {account['username']}") @@ -110,15 +112,24 @@ def create_user(account, tenant_id): def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): + """ + Updates a user with the parameters provided in the user argument + if they are not overwritten by the optional parameters. + :param tenant_id: The target tenant + :type tenant_id: String + :param user: The user as defined in the config, including the username used to identify the user on the system + :param overwrite_name: Optional name to use instead + :type overwrite_name: String + :param overwrite_email: Optional email to use instead + :type overwrite_email: String + :param overwrite_roles: Optional roles to use instead + :type overwrite_roles: List + :param overwrite_pw: Optional password to use instead + :type overwrite_pw: String + :return: response + """ log(f"Trying to update user ... ") - # user_id = user['username'] - # if not name: - # name = user['name'] - # if not email: - # email = user['email'] - # if not roles: - # roles = user['roles'] name = overwrite_name if overwrite_email else user['name'] email = overwrite_email if overwrite_email else user['email'] roles = overwrite_roles if overwrite_roles else user['roles'] @@ -151,16 +162,25 @@ def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, over def __check_api_access(user, tenant_id): - + """ + Checks if the user defined in the config has access to the API. + The check tries to login with the username and password defined in the config, + and sends a get request to '/api/info/me' . + If check fails, asks for user permission to update user. + :param user: The user defined in the config + :type user: Dict + :param tenant_id: The target tenant + :type tenant_id: String + :return: True + """ log(f"Checking API access for user {user['username']}") url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' - headers = {} # {'X-RUN-AS-USER': user['username']} + headers = {} login = { 'user': user['username'], 'password': user['password'] } - try: get_request(url, login, '/api/info/me', headers=headers, use_digest=False) except RequestError: @@ -179,31 +199,47 @@ def __check_api_access(user, tenant_id): print(str(e)) return False - return + return True -def __check_effective_roles(user, tenant_id): +def __check_effective_user_roles(user, tenant_id): + """ + Checks if the effective user roles of the user contain unexpected roles. + prints a warning for each unexpected role. + :param user: User containing the username for whom to retrieve the roles + :type user: Dict + :param tenant_id: The ID of the target tenant + :type tenant_id: String + :return: True + """ log(f"Check effective user roles of user {user['username']}") effective_user_roles = get_user_roles(user['username'], tenant_id) for role in effective_user_roles: for unexpected_role in UNEXPECTED_ROLES: - # ToDo improve this check if role matches unexpected role if unexpected_role in role: - print(f"Unexpected role found for User {user['username']}: {role}") + print(f"WARNING: Unexpected role found for User {user['username']}: {role}") - return + return True -def __check_user_roles(user, tenant_id): +def __check_user_roles(user, existing_user, tenant_id): + """ + Checks if the INTERNAL user roles match the user roles in the config file. + If check fails, asks for user permission to update user. + :param user: The user as defined in the config file + :type user: Dict + :param existing_user: The user as defined on the tenant + :type: Dict + :param tenant_id: The target tenant + :type: String + :return: True + """ log(f"Check user roles of user {user['username']}") - # ToDo change this to exclude group roles - existing_user_roles = get_user_roles(user['username'], tenant_id) + existing_user_roles = extract_internal_user_roles(existing_user) user_roles = user['roles'] - - print('system roles: ', existing_user_roles) - print('config roles: ', user_roles) + log('config roles: ', user_roles) roles = existing_user_roles.copy() missing_roles = [role for role in user_roles if role not in existing_user_roles] @@ -212,6 +248,7 @@ def __check_user_roles(user, tenant_id): if user_roles == existing_user_roles: log('User roles match.') else: + print("existing user roles: ", existing_user_roles) if missing_roles: print("Missing roles: ", missing_roles) action_allowed = check_or_ask_for_permission( @@ -250,34 +287,20 @@ def __check_user_roles(user, tenant_id): if roles != existing_user_roles: # roles = ",".join(roles) - print(roles) update_user(tenant_id, user, overwrite_roles=roles) - return - - -# def get_user_info(user, tenant_id): -# -# url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' -# headers = { -# 'X-RUN-AS-USER': user['username'] -# } -# try: -# response = get_request(url, DIGEST_LOGIN, '/api/info/me', headers=headers) -# except Exception as e: -# log(e) -# return False -# -# return response.json() + return True def get_user_roles(user_name, tenant_id): """ returns the effective roles of a user (user roles + group roles). - Uses DigestLogin - :param user_name: - :param tenant_id: - :return: + Uses DigestLogin. + :param user_name: The username of the user on the tenant + :type user_name: String + :param tenant_id: The traget tenant + :type tenant_id: String + :return: The roles as dict """ url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me/roles' headers = {'X-RUN-AS-USER': user_name} @@ -290,12 +313,34 @@ def get_user_roles(user_name, tenant_id): return response.json() -def get_user(username, tenant_id): - """ sends a GET request to the admin UI to get a user +def extract_internal_user_roles(existing_user, as_string=False): + """ + Extracts the INTERNAL user roles from a user on the tenant. + :param existing_user: The user as defined on the tenant + :type existing_user: JSON + :param as_string: Whether the roles should be returned as a string + :type as_string: Boolean + :return: roles, as list or string + """ + roles = [] + for role in existing_user['roles']: + # ToDo check if 'INTERNAL' is the right thing to use here + if role['type'] == 'INTERNAL': + roles.append(role['name']) + if as_string: + roles = ",".join(sorted(roles)) - :param username: String - :param tenant_id: String - :return: + return roles + + +def get_user(username, tenant_id): + """ + Sends a GET request to the admin UI to get a user + :param username: The username of the user on the tenant + :type username: String + :param tenant_id: The target tenant + :type tenant_id: String + :return: user as JSON """ url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{username}.json' try: @@ -308,4 +353,4 @@ def get_user(username, tenant_id): print(e) return False - return response + return response.json() From 2060de9759d640997bba00a9a476b9d5b3a3a5be Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 22 Jul 2021 18:15:31 +0200 Subject: [PATCH 37/79] Added documentation --- multi-tenant-configuration/configure_users.py | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index abdade6..e97ffee 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -14,6 +14,16 @@ def set_config_users(digest_login, env_conf, config): + """ + Sets/imports the global config variables. + must be called before any checks can be performed. + :param digest_login: The digest login to be used + :type digest_login: DigestLogin + :param env_conf: The environment configuration which specifies the user and system accounts + :type env_conf: dict + :param config: The script configuration + :type config: dict + """ global DIGEST_LOGIN global ENV_CONFIG @@ -22,10 +32,13 @@ def set_config_users(digest_login, env_conf, config): ENV_CONFIG = env_conf CONFIG = config - return - -def check_users(tenant_id): +def check_users(tenant_id: str): + """ + Performs the checks for each user on the specified tenant + :param tenant_id: The target tenant + :type tenant_id: str + """ log('\nStart checking users for tenant: ', tenant_id) # Check and configure System User Accounts & External API User Accounts: @@ -42,7 +55,18 @@ def check_users(tenant_id): check_user(user, tenant_id) -def check_user(user, tenant_id): +def check_user(user: dict, tenant_id: str): + """ + Performs all checks for the specified user: + - checks if user exists + - checks if user has API access (and if password matches) + - checks if the user roles match the roles in the config file + - checks if the user has unexpected roles (effective roles) + :param user: The user to be checked + :type user: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ log(f"Checking user {user['name']} on tenant {tenant_id}.") # Check if user exists @@ -67,6 +91,14 @@ def check_user(user, tenant_id): def __get_roles_as_json_array(account, as_string=False): + """ + Returns the roles of a user account in json format either as a dict or as a string + :param account: User account as defined in the config file + :type account: dict + :param as_string: If the roles should be returned as json string or json object + :type as_string: bool + :return: The roles in json format + """ roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] if as_string: roles = [str(role) for role in roles] @@ -161,7 +193,7 @@ def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, over return response -def __check_api_access(user, tenant_id): +def __check_api_access(user: dict, tenant_id: str) -> bool: """ Checks if the user defined in the config has access to the API. The check tries to login with the username and password defined in the config, @@ -313,17 +345,17 @@ def get_user_roles(user_name, tenant_id): return response.json() -def extract_internal_user_roles(existing_user, as_string=False): +def extract_internal_user_roles(user: dict, as_string=False): """ Extracts the INTERNAL user roles from a user on the tenant. - :param existing_user: The user as defined on the tenant - :type existing_user: JSON + :param user: The user as defined on the tenant + :type user: dict :param as_string: Whether the roles should be returned as a string - :type as_string: Boolean + :type as_string: bool :return: roles, as list or string """ roles = [] - for role in existing_user['roles']: + for role in user['roles']: # ToDo check if 'INTERNAL' is the right thing to use here if role['type'] == 'INTERNAL': roles.append(role['name']) From da377eb9f2d1e2a1ae4fecea616c25a78caf843f Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 22 Jul 2021 18:20:31 +0200 Subject: [PATCH 38/79] minor code restructuring --- multi-tenant-configuration/configure_users.py | 248 +++++++++--------- 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index e97ffee..500a9ca 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -47,15 +47,15 @@ def check_users(tenant_id: str): if organization['id'] == 'dummy': log(f'Checking system accounts for tenant {tenant_id} ...') for system_account in organization['switchcast_system_accounts']: - check_user(system_account, tenant_id) + __check_user(system_account, tenant_id) # check and configure external api accounts if organization['id'] == tenant_id: # ToDo or 'all' ? log(f'Checking External API accounts for tenant {tenant_id} ...') for user in organization['external_api_accounts']: - check_user(user, tenant_id) + __check_user(user, tenant_id) -def check_user(user: dict, tenant_id: str): +def __check_user(user: dict, tenant_id: str): """ Performs all checks for the specified user: - checks if user exists @@ -90,109 +90,6 @@ def check_user(user: dict, tenant_id: str): __check_effective_user_roles(user, tenant_id) -def __get_roles_as_json_array(account, as_string=False): - """ - Returns the roles of a user account in json format either as a dict or as a string - :param account: User account as defined in the config file - :type account: dict - :param as_string: If the roles should be returned as json string or json object - :type as_string: bool - :return: The roles in json format - """ - roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] - if as_string: - roles = [str(role) for role in roles] - roles = '[' + ','.join(roles) + ']' - - return roles - - -def create_user(account, tenant_id): - """ - sends a POST request to the admin UI to create a User - uses the /admin-ng/users/ endpoint - :param account: The user account to be created (e.g. {'username': 'Peter', 'password': '123'} - :type account: dict - :param tenant_id: The target tenant - :type tenant_id: String - :return: response - """ - log(f"Create user {account['username']}") - - url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/' - data = { - 'username': account['username'], - 'password': account['password'], - 'name': account['name'], - 'email': account['email'], - 'roles': __get_roles_as_json_array(account, as_string=True) - } - - try: - response = post_request(url, DIGEST_LOGIN, '/admin-ng/users/', data=data) - except RequestError as err: - if err.get_status_code() == "409": - print(f"Conflict, a user with username {account['username']} already exist.") - elif err.get_status_code() == "403": - print("Forbidden, not enough permissions to create a user with a admin role.") - return False - except Exception as e: - print("User could not be created: ", str(e)) - return False - - return response - - -def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): - """ - Updates a user with the parameters provided in the user argument - if they are not overwritten by the optional parameters. - :param tenant_id: The target tenant - :type tenant_id: String - :param user: The user as defined in the config, including the username used to identify the user on the system - :param overwrite_name: Optional name to use instead - :type overwrite_name: String - :param overwrite_email: Optional email to use instead - :type overwrite_email: String - :param overwrite_roles: Optional roles to use instead - :type overwrite_roles: List - :param overwrite_pw: Optional password to use instead - :type overwrite_pw: String - :return: response - """ - log(f"Trying to update user ... ") - - name = overwrite_name if overwrite_email else user['name'] - email = overwrite_email if overwrite_email else user['email'] - roles = overwrite_roles if overwrite_roles else user['roles'] - pw = overwrite_pw if overwrite_pw else user['password'] - if not isinstance(roles, list): # in case only one role is given, make sure roles is a list - roles = [roles] - roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) - - url = f"{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user['username']}.json" - data = { - 'password': pw, - 'name': name, - 'email': email, - 'roles': roles - } - try: - response = put_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json', data=data) - except RequestError as err: - print("RequestError: ", err) - if err.get_status_code() == "400": - print(f"Bad Request: Invalid data provided.") - return False - except Exception as e: - print(f"User with name {name} could not be updated. \n", "Exception: ", str(e)) - return False - - log(f"Updated user {name}.") - - return response - - def __check_api_access(user: dict, tenant_id: str) -> bool: """ Checks if the user defined in the config has access to the API. @@ -324,6 +221,115 @@ def __check_user_roles(user, existing_user, tenant_id): return True +def get_user(username, tenant_id): + """ + Sends a GET request to the admin UI to get a user + :param username: The username of the user on the tenant + :type username: String + :param tenant_id: The target tenant + :type tenant_id: String + :return: user as JSON + """ + url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{username}.json' + try: + response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') + except RequestError as err: + if not err.get_status_code() == "404": + print(err) + return False + except Exception as e: + print(e) + return False + + return response.json() + + +def create_user(account, tenant_id): + """ + sends a POST request to the admin UI to create a User + uses the /admin-ng/users/ endpoint + :param account: The user account to be created (e.g. {'username': 'Peter', 'password': '123'} + :type account: dict + :param tenant_id: The target tenant + :type tenant_id: String + :return: response + """ + log(f"Create user {account['username']}") + + url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/' + data = { + 'username': account['username'], + 'password': account['password'], + 'name': account['name'], + 'email': account['email'], + 'roles': __get_roles_as_json_array(account, as_string=True) + } + + try: + response = post_request(url, DIGEST_LOGIN, '/admin-ng/users/', data=data) + except RequestError as err: + if err.get_status_code() == "409": + print(f"Conflict, a user with username {account['username']} already exist.") + elif err.get_status_code() == "403": + print("Forbidden, not enough permissions to create a user with a admin role.") + return False + except Exception as e: + print("User could not be created: ", str(e)) + return False + + return response + + +def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): + """ + Updates a user with the parameters provided in the user argument + if they are not overwritten by the optional parameters. + :param tenant_id: The target tenant + :type tenant_id: String + :param user: The user as defined in the config, including the username used to identify the user on the system + :param overwrite_name: Optional name to use instead + :type overwrite_name: String + :param overwrite_email: Optional email to use instead + :type overwrite_email: String + :param overwrite_roles: Optional roles to use instead + :type overwrite_roles: List + :param overwrite_pw: Optional password to use instead + :type overwrite_pw: String + :return: response + """ + log(f"Trying to update user ... ") + + name = overwrite_name if overwrite_email else user['name'] + email = overwrite_email if overwrite_email else user['email'] + roles = overwrite_roles if overwrite_roles else user['roles'] + pw = overwrite_pw if overwrite_pw else user['password'] + if not isinstance(roles, list): # in case only one role is given, make sure roles is a list + roles = [roles] + roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) + + url = f"{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user['username']}.json" + data = { + 'password': pw, + 'name': name, + 'email': email, + 'roles': roles + } + try: + response = put_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json', data=data) + except RequestError as err: + print("RequestError: ", err) + if err.get_status_code() == "400": + print(f"Bad Request: Invalid data provided.") + return False + except Exception as e: + print(f"User with name {name} could not be updated. \n", "Exception: ", str(e)) + return False + + log(f"Updated user {name}.") + + return response + + def get_user_roles(user_name, tenant_id): """ returns the effective roles of a user (user roles + group roles). @@ -365,24 +371,18 @@ def extract_internal_user_roles(user: dict, as_string=False): return roles -def get_user(username, tenant_id): +def __get_roles_as_json_array(account, as_string=False): """ - Sends a GET request to the admin UI to get a user - :param username: The username of the user on the tenant - :type username: String - :param tenant_id: The target tenant - :type tenant_id: String - :return: user as JSON + Returns the roles of a user account in json format either as a dict or as a string + :param account: User account as defined in the config file + :type account: dict + :param as_string: If the roles should be returned as json string or json object + :type as_string: bool + :return: The roles in json format """ - url = f'{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{username}.json' - try: - response = get_request(url, DIGEST_LOGIN, '/admin-ng/users/{username}.json') - except RequestError as err: - if not err.get_status_code() == "404": - print(err) - return False - except Exception as e: - print(e) - return False + roles = [{'name': role, 'type': 'INTERNAL'} for role in account['roles']] + if as_string: + roles = [str(role) for role in roles] + roles = '[' + ','.join(roles) + ']' - return response.json() + return roles From 7fbe1eea5d057249905c4f96cfa09b9d1431f6db Mon Sep 17 00:00:00 2001 From: mheyen Date: Thu, 22 Jul 2021 18:28:43 +0200 Subject: [PATCH 39/79] added function parameter types --- multi-tenant-configuration/configure_users.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 500a9ca..b16942b 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -131,7 +131,7 @@ def __check_api_access(user: dict, tenant_id: str) -> bool: return True -def __check_effective_user_roles(user, tenant_id): +def __check_effective_user_roles(user: dict, tenant_id: str): """ Checks if the effective user roles of the user contain unexpected roles. prints a warning for each unexpected role. @@ -152,7 +152,7 @@ def __check_effective_user_roles(user, tenant_id): return True -def __check_user_roles(user, existing_user, tenant_id): +def __check_user_roles(user: dict, existing_user: dict, tenant_id: str): """ Checks if the INTERNAL user roles match the user roles in the config file. If check fails, asks for user permission to update user. @@ -221,7 +221,7 @@ def __check_user_roles(user, existing_user, tenant_id): return True -def get_user(username, tenant_id): +def get_user(username: str, tenant_id: str): """ Sends a GET request to the admin UI to get a user :param username: The username of the user on the tenant @@ -244,7 +244,7 @@ def get_user(username, tenant_id): return response.json() -def create_user(account, tenant_id): +def create_user(account: dict, tenant_id: str): """ sends a POST request to the admin UI to create a User uses the /admin-ng/users/ endpoint @@ -280,7 +280,8 @@ def create_user(account, tenant_id): return response -def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): +def update_user(tenant_id: str, user: dict, + overwrite_name=None, overwrite_email=None, overwrite_roles=None, overwrite_pw=None): """ Updates a user with the parameters provided in the user argument if they are not overwritten by the optional parameters. @@ -330,7 +331,7 @@ def update_user(tenant_id, user, overwrite_name=None, overwrite_email=None, over return response -def get_user_roles(user_name, tenant_id): +def get_user_roles(user_name: str, tenant_id: str): """ returns the effective roles of a user (user roles + group roles). Uses DigestLogin. @@ -371,7 +372,7 @@ def extract_internal_user_roles(user: dict, as_string=False): return roles -def __get_roles_as_json_array(account, as_string=False): +def __get_roles_as_json_array(account: dict, as_string=False): """ Returns the roles of a user account in json format either as a dict or as a string :param account: User account as defined in the config file From 5670ab6456aa82a732620b662c208b001e7e417f Mon Sep 17 00:00:00 2001 From: mheyen Date: Fri, 23 Jul 2021 01:02:15 +0200 Subject: [PATCH 40/79] added code documentation and improved readability --- .../configure_groups.py | 284 ++++++++++++------ 1 file changed, 198 insertions(+), 86 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index fdf1d9c..c2f1d29 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -1,5 +1,6 @@ from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError +from args.digest_login import DigestLogin from configure_users import get_user from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission @@ -11,7 +12,17 @@ DIGEST_LOGIN = None -def set_config_groups(digest_login, group_config, config): +def set_config_groups(digest_login: DigestLogin, group_config: dict, config: dict): + """ + Sets/imports the global config variables. + Must be called before any checks can be performed. + :param digest_login: The digest login to be used + :type digest_login: DigestLogin + :param group_config: The group configuration which specifies the groups on each tenant + :type group_config: dict + :param config: The script configuration + :type config: dict + """ global DIGEST_LOGIN global GROUP_CONFIG @@ -20,25 +31,41 @@ def set_config_groups(digest_login, group_config, config): GROUP_CONFIG = group_config CONFIG = config - return - -def check_groups(tenant_id): +def check_groups(tenant_id: str): + """ + Performs the checks for each group on the specified tenant + :param tenant_id: The target tenant + :type tenant_id: str + """ log('\nStart checking groups for tenant: ', tenant_id) # For all Groups: for group in GROUP_CONFIG['groups']: # Check group if group['tenants'] == 'all' or group['tenants'] == tenant_id: - group['identifier'] = generate_group_identifier(group, tenant_id) - check_group(group=group, tenant_id=tenant_id) + group['identifier'] = __generate_group_identifier(group, tenant_id) + __check_group(group=group, tenant_id=tenant_id) -def check_group(group, tenant_id): +def __check_group(group: dict, tenant_id: str): + """ + Performs all checks for the specified group: + - checks if group exists + - checks if group description matches + - checks if group members match + - checks if group roles match + - ToDo Check API access of all group members + - ToDo Check group type + :param group: The group to be checked + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ log(f"\nCheck group {group['name']} with id {group['identifier']}") # Check if group exists. - existing_group = check_if_group_exists(group, tenant_id) + existing_group = get_group(group, tenant_id) if not existing_group: # Create group if it does not exist. # Ask for permission @@ -53,21 +80,29 @@ def check_group(group, tenant_id): else: # Check if group name and description match the name and description provided in the configuration. # Update them if they do not match. (Asks for permission) - check_group_description(group=group, existing_group=existing_group, tenant_id=tenant_id) + __check_group_description(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group members exist. # Check if group members match the group members provided in the configuration. # Add or remove members accordingly. - check_group_members(group=group, existing_group=existing_group, tenant_id=tenant_id) + __check_group_members(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. # Update group roles if they do not match.(Asks for permission) - check_group_roles(group=group, existing_group=existing_group, tenant_id=tenant_id) - + __check_group_roles(group=group, existing_group=existing_group, tenant_id=tenant_id) + # ToDo # Check external API accounts of members. Add missing API accounts. # Check group type. If group is closed, remove unexpected members. # Update group members. (Asks for permission) -def check_if_group_exists(group, tenant_id): +def get_group(group: dict, tenant_id: str): + """ + Checks if the group exists on the specified tenant + :param group: The group as defined in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :return: The group as specified on the tenant if it exists or False + """ log(f"check if group {group['name']} exists.") url = f"{CONFIG.tenant_urls[tenant_id]}/api/groups/{group['identifier']}" @@ -76,22 +111,33 @@ def check_if_group_exists(group, tenant_id): return response.json() except RequestError as err: if err.get_status_code() == "404": - return False + pass else: raise Exception except Exception as e: print("ERROR: ", str(e)) - return False + return False -def check_group_description(group, existing_group, tenant_id): +def __check_group_description(group: dict, existing_group: dict, tenant_id: str): + """ + Checks if the group description matches. + Ask for permission to update the group if necessary. + :param group: The group as specified in the config file + :type group: dict + :param existing_group: The existing group as specified on the tenant system + :type existing_group: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ log(f"check names and description for group {group['name']}.") + # ToDo: does it really makes sense to check for the name? # This seems to be already done when checking for the existence of the group. if group['name'] != existing_group['name']: print("WARNING: Group names do not match. ") return - if group_description_template(group['description'], tenant_id) == existing_group['description']: + if __group_description_template(group['description'], tenant_id) == existing_group['description']: log('Group descriptions match.') else: action_allowed = check_or_ask_for_permission( @@ -107,17 +153,23 @@ def check_group_description(group, existing_group, tenant_id): name=group['name'] ) - return - -def check_group_members(group, existing_group, tenant_id): +def __check_group_members(group: dict, existing_group: dict, tenant_id: str): + """ + Checks if the group member match. + Asks for permission to either add or remove members accordingly. + :param group: The group as specified in the configuration file + :type group: dict + :param existing_group: The existing group as specified on the tenant system + :type existing_group: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ log(f"Check members for group {group['name']}.") - group_members = extract_members_from_group(group=group, tenant_id=tenant_id) + group_members = __extract_members_from_group(group=group, tenant_id=tenant_id) existing_group_members = sorted(filter(None, existing_group['members'].split(","))) - log("Config group members: ", group_members) - log("Existing group members: ", existing_group_members) members = existing_group_members.copy() missing_members = [member for member in group_members if member not in existing_group_members] @@ -130,6 +182,7 @@ def check_group_members(group, existing_group, tenant_id): if not missing_members and not additional_members: log('Group members match.') else: + print("Existing group members: ", existing_group_members) if missing_members: print("Missing members: ", missing_members) action_allowed = check_or_ask_for_permission( @@ -172,17 +225,23 @@ def check_group_members(group, existing_group, tenant_id): members = ",".join(members) update_group(tenant_id=tenant_id, group=group, members=members) - return - -def check_group_roles(group, existing_group, tenant_id): +def __check_group_roles(group: dict, existing_group: dict, tenant_id: str): + """ + Checks if the group roles match. + Asks for permission to either add or remove roles accordingly. + :param group: The group as specified in the configuration file + :type group: dict + :param existing_group: The existing group as specified on the tenant system + :type existing_group: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ log(f"Check roles for group {group['name']}.") - group_roles = extract_roles_from_group(group=group, tenant_id=tenant_id).split(",") + group_roles = __extract_roles_from_group(group=group, tenant_id=tenant_id) existing_group_roles = sorted(existing_group['roles'].split(",")) - log("Config group roles: ", group_roles) - log("Existing group roles: ", existing_group_roles) roles = existing_group_roles.copy() missing_roles = [role for role in group_roles if role not in existing_group_roles] @@ -191,6 +250,7 @@ def check_group_roles(group, existing_group, tenant_id): if group_roles == existing_group_roles: log('Group roles match.') else: + print("Existing group roles: ", existing_group_roles) if missing_roles: print("Missing roles: ", missing_roles) action_allowed = check_or_ask_for_permission( @@ -235,33 +295,30 @@ def check_group_roles(group, existing_group, tenant_id): return -def generate_group_identifier(group, tenant_id): +def __generate_group_identifier(group: dict, tenant_id: str): + """ + generates the group identifier based on the group name + :param group: The group as specified in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :return: The group id: str + """ # ToDo check if the generated identifiers are correct! (the same as in the ruby script) # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() return group['name'].replace(' ', '_').lower() -def get_groups_from_tenant(tenant_id): - - url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' - try: - response = get_request(url, DIGEST_LOGIN, '/api/groups/') - except RequestError as err: - print('RequestError: ', err) - return False - except Exception as e: - print("Groups could not be retrieved. \n", "Error: ", str(e)) - return False - - return response.json() - - -def extract_roles_from_group(group, tenant_id): +def __extract_roles_from_group(group: dict, tenant_id: str, as_string=False): """ - - :param group: - :param tenant_id: - :return: sorted comma separated list of roles (e.g. "ROLE_ADMIN,ROLE_SUDO") + Parses the group configuration and extracts the tenant specific group roles for a specific group. + :param group: The group as specified in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :param as_string: Whether the roles should be returned as a string or a list + :type as_string: bool + :return: Sorted comma separated list of roles (e.g. "ROLE_ADMIN,ROLE_SUDO" or ['ROLE_ADMIN', 'ROLE_SUDO'] ) """ roles = [] for permission in group['permissions']: @@ -276,60 +333,94 @@ def extract_roles_from_group(group, tenant_id): for role in permission['roles']['remove']: if role in roles: roles.remove(role) - roles = ','.join(sorted(roles)) + if as_string: + roles = ','.join(sorted(roles)) return roles -def extract_members_from_group(group, tenant_id, as_string=False): +def __extract_members_from_group(group: dict, tenant_id: str, as_string=False): """ - Does not check if member exists on tenant - :param group: - :param tenant_id: - :param as_string: + Parses the group configuration and extracts the tenant specific group members. + Does not check if a member exists on the tenant. + :param group: The group as specified in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :param as_string: Whether the roles should be returned as a string or a list + :type as_string: bool :return: Comma separated string of members (e.g. "guy1,guy2") or list of members. """ members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] if as_string: members = ",".join(sorted(members)) - return members -def group_description_template(description, tenant_id): +def __group_description_template(description: str, tenant_id: str): + """ + replaces placeholders for names in the group description + :param description: The group description with placeholders + :type description: str + :param tenant_id: The tenant id to be inserted into the description + :type tenant_id: str + :return: group description with the inserted name, str + """ # ToDo check for a better way to insert into template description = description.replace("${name}", tenant_id) return description -def update_group(tenant_id, group=None, name=None, description=None, roles=None, members=None): - log(f"Try to update group ... ") - - if not name and not group: - log("Cannot update group without a specified name.") - return False +def update_group(tenant_id: str, group: dict, + overwrite_name=None, overwrite_description=None, overwrite_roles=None, overwrite_members=None): + """ + Updates the group on the tenant. + Either with the parameters defined in the group or with the specific parameters to individually overwrite them. + :param tenant_id: The target tenant + :type tenant_id: str + :param group: The group as specified in the configuration file + :type group: dict + :param overwrite_name: Optional name + :type overwrite_name: str or None + :param overwrite_description: Optional description + :type overwrite_description: str or None + :param overwrite_roles: Optional roles + :type overwrite_roles: str or None + :param overwrite_members: Optional members + :type overwrite_members: str or None + :return: Returns the response if successful or False if the request failed + """ + log(f"Trying to update group ... ") + + group_id = group['identifier'] + name = overwrite_name if overwrite_name else group['name'] + description = overwrite_description if overwrite_description else \ + __group_description_template(group['description'], tenant_id) + roles = overwrite_roles if overwrite_roles else \ + __extract_roles_from_group(group, tenant_id, as_string=True) + members = overwrite_members if overwrite_members else \ + __extract_members_from_group(group, tenant_id, as_string=True) + + # if group: + # group_id = group['identifier'] + # if not name: + # name = group['name'] + # if not members: + # members = __extract_members_from_group(group, tenant_id, as_string=True) + # if not roles: + # roles = __extract_roles_from_group(group, tenant_id, as_string=True) + # if not description: + # description = __group_description_template(group['description'], tenant_id) + # else: + # group_id = __generate_group_identifier(group={'name': name}, tenant_id=tenant_id) - if group: - group_id = group['identifier'] - if not name: - name = group['name'] - if not members: - members = extract_members_from_group(group, tenant_id, as_string=True) - if not roles: - roles = extract_roles_from_group(group, tenant_id) - if not description: - description = group_description_template(group['description'], tenant_id) - else: - group_id = generate_group_identifier(group={'name': name}, tenant_id=tenant_id) url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/{group_id}' - data = { 'name': name, 'description': description, 'roles': roles, 'members': members, } - try: response = put_request(url, DIGEST_LOGIN, '/api/groups/{groupId}', data=data) except RequestError as err: @@ -342,32 +433,38 @@ def update_group(tenant_id, group=None, name=None, description=None, roles=None, return False log(f"Updated group {name}.") - return response -def create_group(group, tenant_id): +def create_group(group: dict, tenant_id: str): + """ + Sends a POST request to /api/groups/ to create a new group with the given parameter. + :param group: The group to be created (usually the one specified in the configuration file) + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :return: Returns the response if successful or False if the request failed + """ log(f"trying to create group {group['name']}. ") url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' # extract members and roles - members = extract_members_from_group(group, tenant_id) + members = __extract_members_from_group(group, tenant_id) # check if member exist on tenant for member in members: if not get_user(username=member, tenant_id=tenant_id): - print(f"Member {member} does not exist.") + print(f"Warning: Member {member} does not exist.") members.remove(member) members = ",".join(members) - roles = extract_roles_from_group(group, tenant_id) - description = group_description_template(group['description'], tenant_id) + roles = __extract_roles_from_group(group, tenant_id, as_string=True) + data = { 'name': group['name'], - 'description': description, + 'description': __group_description_template(group['description'], tenant_id), 'roles': roles, 'members': members, } - try: response = post_request(url, DIGEST_LOGIN, '/api/groups/', data=data) except RequestError as err: @@ -384,3 +481,18 @@ def create_group(group, tenant_id): log(f"created group {group['name']}.\nmembers: {members} \nroles: {roles} ") return response + + +# def get_groups_from_tenant(tenant_id): +# +# url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' +# try: +# response = get_request(url, DIGEST_LOGIN, '/api/groups/') +# except RequestError as err: +# print('RequestError: ', err) +# return False +# except Exception as e: +# print("Groups could not be retrieved. \n", "Error: ", str(e)) +# return False +# +# return response.json() From 54293684e6321d73c62cf30a8d47f100fa931396 Mon Sep 17 00:00:00 2001 From: mheyen Date: Fri, 23 Jul 2021 01:03:06 +0200 Subject: [PATCH 41/79] fixed typo and added missing documentation --- multi-tenant-configuration/configure_users.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index b16942b..8b15476 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -1,5 +1,6 @@ from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError +from args.digest_login import DigestLogin from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission from parsing_configurations import __abort_script, log @@ -13,7 +14,7 @@ UNEXPECTED_ROLES = ["ROLE_ADMIN", "ROLE_ADMIN_UI", "ROLE_UI_", "ROLE_CAPTURE_"] -def set_config_users(digest_login, env_conf, config): +def set_config_users(digest_login: DigestLogin, env_conf: dict, config: dict): """ Sets/imports the global config variables. must be called before any checks can be performed. @@ -300,12 +301,14 @@ def update_user(tenant_id: str, user: dict, """ log(f"Trying to update user ... ") - name = overwrite_name if overwrite_email else user['name'] + name = overwrite_name if overwrite_name else user['name'] email = overwrite_email if overwrite_email else user['email'] roles = overwrite_roles if overwrite_roles else user['roles'] pw = overwrite_pw if overwrite_pw else user['password'] - if not isinstance(roles, list): # in case only one role is given, make sure roles is a list - roles = [roles] + # if not isinstance(roles, list): # in case only one role is given, make sure roles is a list + # roles = [roles] + # in case only one role is given, make sure roles is a list + roles = roles if isinstance(roles, list) else [roles] roles = __get_roles_as_json_array(account={'roles': roles}, as_string=True) url = f"{CONFIG.tenant_urls[tenant_id]}/admin-ng/users/{user['username']}.json" @@ -327,7 +330,6 @@ def update_user(tenant_id: str, user: dict, return False log(f"Updated user {name}.") - return response From 27126474efab02b9526df60d9d82598250fae018 Mon Sep 17 00:00:00 2001 From: mheyen Date: Fri, 23 Jul 2021 01:08:09 +0200 Subject: [PATCH 42/79] minor code restructuring --- .../configure_groups.py | 238 +++++++++--------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index c2f1d29..86260a6 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -94,31 +94,6 @@ def __check_group(group: dict, tenant_id: str): # Update group members. (Asks for permission) -def get_group(group: dict, tenant_id: str): - """ - Checks if the group exists on the specified tenant - :param group: The group as defined in the configuration file - :type group: dict - :param tenant_id: The target tenant - :type tenant_id: str - :return: The group as specified on the tenant if it exists or False - """ - log(f"check if group {group['name']} exists.") - - url = f"{CONFIG.tenant_urls[tenant_id]}/api/groups/{group['identifier']}" - try: - response = get_request(url, DIGEST_LOGIN, '/api/groups/') - return response.json() - except RequestError as err: - if err.get_status_code() == "404": - pass - else: - raise Exception - except Exception as e: - print("ERROR: ", str(e)) - return False - - def __check_group_description(group: dict, existing_group: dict, tenant_id: str): """ Checks if the group description matches. @@ -295,80 +270,76 @@ def __check_group_roles(group: dict, existing_group: dict, tenant_id: str): return -def __generate_group_identifier(group: dict, tenant_id: str): +def get_group(group: dict, tenant_id: str): """ - generates the group identifier based on the group name - :param group: The group as specified in the configuration file + Checks if the group exists on the specified tenant + :param group: The group as defined in the configuration file :type group: dict :param tenant_id: The target tenant :type tenant_id: str - :return: The group id: str + :return: The group as specified on the tenant if it exists or False """ - # ToDo check if the generated identifiers are correct! (the same as in the ruby script) - # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() - return group['name'].replace(' ', '_').lower() - + log(f"check if group {group['name']} exists.") -def __extract_roles_from_group(group: dict, tenant_id: str, as_string=False): - """ - Parses the group configuration and extracts the tenant specific group roles for a specific group. - :param group: The group as specified in the configuration file - :type group: dict - :param tenant_id: The target tenant - :type tenant_id: str - :param as_string: Whether the roles should be returned as a string or a list - :type as_string: bool - :return: Sorted comma separated list of roles (e.g. "ROLE_ADMIN,ROLE_SUDO" or ['ROLE_ADMIN', 'ROLE_SUDO'] ) - """ - roles = [] - for permission in group['permissions']: - # add all default roles - if permission['tenants'] == 'all': - for role in permission['roles']: - roles.append(role) - # add/remove tenant specific roles - elif permission['tenants'] == tenant_id: - for role in permission['roles']['add']: - roles.append(role) - for role in permission['roles']['remove']: - if role in roles: - roles.remove(role) - if as_string: - roles = ','.join(sorted(roles)) - return roles + url = f"{CONFIG.tenant_urls[tenant_id]}/api/groups/{group['identifier']}" + try: + response = get_request(url, DIGEST_LOGIN, '/api/groups/') + return response.json() + except RequestError as err: + if err.get_status_code() == "404": + pass + else: + raise Exception + except Exception as e: + print("ERROR: ", str(e)) + return False -def __extract_members_from_group(group: dict, tenant_id: str, as_string=False): +def create_group(group: dict, tenant_id: str): """ - Parses the group configuration and extracts the tenant specific group members. - Does not check if a member exists on the tenant. - :param group: The group as specified in the configuration file + Sends a POST request to /api/groups/ to create a new group with the given parameter. + :param group: The group to be created (usually the one specified in the configuration file) :type group: dict :param tenant_id: The target tenant :type tenant_id: str - :param as_string: Whether the roles should be returned as a string or a list - :type as_string: bool - :return: Comma separated string of members (e.g. "guy1,guy2") or list of members. + :return: Returns the response if successful or False if the request failed """ - members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] - if as_string: - members = ",".join(sorted(members)) - return members + log(f"trying to create group {group['name']}. ") + url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' -def __group_description_template(description: str, tenant_id: str): - """ - replaces placeholders for names in the group description - :param description: The group description with placeholders - :type description: str - :param tenant_id: The tenant id to be inserted into the description - :type tenant_id: str - :return: group description with the inserted name, str - """ - # ToDo check for a better way to insert into template - description = description.replace("${name}", tenant_id) + # extract members and roles + members = __extract_members_from_group(group, tenant_id) + # check if member exist on tenant + for member in members: + if not get_user(username=member, tenant_id=tenant_id): + print(f"Warning: Member {member} does not exist.") + members.remove(member) + members = ",".join(members) + roles = __extract_roles_from_group(group, tenant_id, as_string=True) - return description + data = { + 'name': group['name'], + 'description': __group_description_template(group['description'], tenant_id), + 'roles': roles, + 'members': members, + } + try: + response = post_request(url, DIGEST_LOGIN, '/api/groups/', data=data) + except RequestError as err: + if err.get_status_code() == "400": + print(f"Bad Request: Group with name {group['name']} could not be created.") + elif err.get_status_code() == "409": + print(f"Conflict: Group with name {group['name']} could not be created.\n" + f"Potentially, Group with name {group['name']} already exists.") + print("RequestError: ", err) + return False + except Exception as e: + print(f"Group with name {group['name']} could not be created. \n", "Exception: ", str(e)) + return False + + log(f"created group {group['name']}.\nmembers: {members} \nroles: {roles} ") + return response def update_group(tenant_id: str, group: dict, @@ -436,51 +407,80 @@ def update_group(tenant_id: str, group: dict, return response -def create_group(group: dict, tenant_id: str): +def __generate_group_identifier(group: dict, tenant_id: str): """ - Sends a POST request to /api/groups/ to create a new group with the given parameter. - :param group: The group to be created (usually the one specified in the configuration file) + generates the group identifier based on the group name + :param group: The group as specified in the configuration file :type group: dict :param tenant_id: The target tenant :type tenant_id: str - :return: Returns the response if successful or False if the request failed + :return: The group id: str """ - log(f"trying to create group {group['name']}. ") + # ToDo check if the generated identifiers are correct! (the same as in the ruby script) + # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() + return group['name'].replace(' ', '_').lower() - url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' - # extract members and roles - members = __extract_members_from_group(group, tenant_id) - # check if member exist on tenant - for member in members: - if not get_user(username=member, tenant_id=tenant_id): - print(f"Warning: Member {member} does not exist.") - members.remove(member) - members = ",".join(members) - roles = __extract_roles_from_group(group, tenant_id, as_string=True) +def __group_description_template(description: str, tenant_id: str): + """ + replaces placeholders for names in the group description + :param description: The group description with placeholders + :type description: str + :param tenant_id: The tenant id to be inserted into the description + :type tenant_id: str + :return: group description with the inserted name, str + """ + # ToDo check for a better way to insert into template + description = description.replace("${name}", tenant_id) - data = { - 'name': group['name'], - 'description': __group_description_template(group['description'], tenant_id), - 'roles': roles, - 'members': members, - } - try: - response = post_request(url, DIGEST_LOGIN, '/api/groups/', data=data) - except RequestError as err: - if err.get_status_code() == "400": - print(f"Bad Request: Group with name {group['name']} could not be created.") - elif err.get_status_code() == "409": - print(f"Conflict: Group with name {group['name']} could not be created.\n" - f"Potentially, Group with name {group['name']} already exists.") - print("RequestError: ", err) - return False - except Exception as e: - print(f"Group with name {group['name']} could not be created. \n", "Exception: ", str(e)) - return False + return description - log(f"created group {group['name']}.\nmembers: {members} \nroles: {roles} ") - return response + +def __extract_members_from_group(group: dict, tenant_id: str, as_string=False): + """ + Parses the group configuration and extracts the tenant specific group members. + Does not check if a member exists on the tenant. + :param group: The group as specified in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :param as_string: Whether the roles should be returned as a string or a list + :type as_string: bool + :return: Comma separated string of members (e.g. "guy1,guy2") or list of members. + """ + members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] + if as_string: + members = ",".join(sorted(members)) + return members + + +def __extract_roles_from_group(group: dict, tenant_id: str, as_string=False): + """ + Parses the group configuration and extracts the tenant specific group roles for a specific group. + :param group: The group as specified in the configuration file + :type group: dict + :param tenant_id: The target tenant + :type tenant_id: str + :param as_string: Whether the roles should be returned as a string or a list + :type as_string: bool + :return: Sorted comma separated list of roles (e.g. "ROLE_ADMIN,ROLE_SUDO" or ['ROLE_ADMIN', 'ROLE_SUDO'] ) + """ + roles = [] + for permission in group['permissions']: + # add all default roles + if permission['tenants'] == 'all': + for role in permission['roles']: + roles.append(role) + # add/remove tenant specific roles + elif permission['tenants'] == tenant_id: + for role in permission['roles']['add']: + roles.append(role) + for role in permission['roles']['remove']: + if role in roles: + roles.remove(role) + if as_string: + roles = ','.join(sorted(roles)) + return roles # def get_groups_from_tenant(tenant_id): From 265c01a2d8ac4150006e4245f8a42e5a556459d8 Mon Sep 17 00:00:00 2001 From: mheyen Date: Fri, 23 Jul 2021 01:18:36 +0200 Subject: [PATCH 43/79] fixed update_group function calls --- multi-tenant-configuration/configure_groups.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 86260a6..afb9e9e 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -122,11 +122,7 @@ def __check_group_description(group: dict, existing_group: dict, tenant_id: str) tenant_id=tenant_id ) if action_allowed: - update_group( - tenant_id=tenant_id, - description=group['description'], - name=group['name'] - ) + update_group(tenant_id, group) def __check_group_members(group: dict, existing_group: dict, tenant_id: str): @@ -198,7 +194,7 @@ def __check_group_members(group: dict, existing_group: dict, tenant_id: str): if members != existing_group_members: # members = ",".join(list(dict.fromkeys(members))) members = ",".join(members) - update_group(tenant_id=tenant_id, group=group, members=members) + update_group(tenant_id, group, overwrite_members=members) def __check_group_roles(group: dict, existing_group: dict, tenant_id: str): @@ -265,9 +261,7 @@ def __check_group_roles(group: dict, existing_group: dict, tenant_id: str): if roles != existing_group_roles: # roles = ",".join(list(dict.fromkeys(roles))) roles = ",".join(roles) - update_group(tenant_id=tenant_id, group=group, roles=roles) - - return + update_group(tenant_id, group, overwrite_roles=roles) def get_group(group: dict, tenant_id: str): From 0987a53b5b39530a941451a70ad1f66259757ccc Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 28 Jul 2021 20:52:17 +0200 Subject: [PATCH 44/79] Removed comments and added documentation --- multi-tenant-configuration/config.py | 4 +-- .../configure_groups.py | 32 ++----------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 3e836f0..5989940 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -3,7 +3,7 @@ # Set this to your global admin node base_url = "http://localhost:8080" -# If you have multiple tenants use an URL pattern. +# If you have multiple tenants, use an URL pattern. # example: # tenant_url_pattern = "https://{}.example.org" tenant_url_pattern = "http://{}:8080" @@ -16,7 +16,7 @@ # } # tenant_urls = {'tenant1': 'https://develop.opencast.org'} -# digest user +# Digest User login digest_user = "opencast_system_account" digest_pw = "CHANGE_ME" diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index afb9e9e..b143d07 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -86,11 +86,13 @@ def __check_group(group: dict, tenant_id: str): # Add or remove members accordingly. __check_group_members(group=group, existing_group=existing_group, tenant_id=tenant_id) # Check if group roles match the group roles provided in the configuration. - # Update group roles if they do not match.(Asks for permission) + # Update group roles if they do not match. (Asks for permission) __check_group_roles(group=group, existing_group=existing_group, tenant_id=tenant_id) # ToDo # Check external API accounts of members. Add missing API accounts. + # ToDo should this actually be done? We already ask if members should be removed in the member check. # Check group type. If group is closed, remove unexpected members. + # Check for invalid group type # Update group members. (Asks for permission) @@ -366,19 +368,6 @@ def update_group(tenant_id: str, group: dict, members = overwrite_members if overwrite_members else \ __extract_members_from_group(group, tenant_id, as_string=True) - # if group: - # group_id = group['identifier'] - # if not name: - # name = group['name'] - # if not members: - # members = __extract_members_from_group(group, tenant_id, as_string=True) - # if not roles: - # roles = __extract_roles_from_group(group, tenant_id, as_string=True) - # if not description: - # description = __group_description_template(group['description'], tenant_id) - # else: - # group_id = __generate_group_identifier(group={'name': name}, tenant_id=tenant_id) - url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/{group_id}' data = { 'name': name, @@ -475,18 +464,3 @@ def __extract_roles_from_group(group: dict, tenant_id: str, as_string=False): if as_string: roles = ','.join(sorted(roles)) return roles - - -# def get_groups_from_tenant(tenant_id): -# -# url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/' -# try: -# response = get_request(url, DIGEST_LOGIN, '/api/groups/') -# except RequestError as err: -# print('RequestError: ', err) -# return False -# except Exception as e: -# print("Groups could not be retrieved. \n", "Error: ", str(e)) -# return False -# -# return response.json() From fd83e525251ccd53b4468240d8dc44251f3a294a Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 3 Aug 2021 23:10:07 +0200 Subject: [PATCH 45/79] Added check for Capture Agent Accounts This check will test if the capture agents defined in the organization config have access to the ingest service and if the password is correct. --- .../configure_capture_accounts.py | 103 ++++++++++++++++++ multi-tenant-configuration/main.py | 13 ++- 2 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 multi-tenant-configuration/configure_capture_accounts.py diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py new file mode 100644 index 0000000..76e554f --- /dev/null +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -0,0 +1,103 @@ +from rest_requests.request import get_request +from rest_requests.request_error import RequestError +from args.digest_login import DigestLogin +from parsing_configurations import log + + +CONFIG = None +ENV_CONFIG = None +DIGEST_LOGIN = None + + +def set_config_capture_accounts(digest_login: DigestLogin, env_conf: dict, config: dict): + """ + Sets/imports the global config variables. + must be called before any checks can be performed. + :param digest_login: The digest login to be used + :type digest_login: DigestLogin + :param env_conf: The environment configuration which specifies the user and system accounts + :type env_conf: dict + :param config: The script configuration + :type config: dict + """ + + global DIGEST_LOGIN + global ENV_CONFIG + global CONFIG + DIGEST_LOGIN = digest_login + ENV_CONFIG = env_conf + CONFIG = config + + +def check_capture_accounts(tenant_id: str): + """ + Performs the checks for each capture agent on the specified tenant + :param tenant_id: The target tenant + :type tenant_id: str + """ + log('\nStart checking Capture Agent Accounts for tenant: ', tenant_id) + + # Check and configure Capture Agent Accounts: + for organization in ENV_CONFIG['opencast_organizations']: + # check switchcast system accounts + if organization['id'] == tenant_id: + for capture_agent in organization['capture_agent_accounts']: + __check_capture_agent_account(capture_agent, tenant_id) + + +def __check_capture_agent_account(account: dict, tenant_id: str): + """ + Performs all checks for the specified Capture Agent Account: + - checks if account has API access (and if password matches) + - checks if username and password exists + :param account: The Capture Agent Account to be checked + :type account: dict + :param tenant_id: The target tenant + :type tenant_id: str + """ + log(f"Checking Capture Agent Account {account['username']} on tenant {tenant_id}.") + + # check username and password + if not account['username']: + print('WARNING: No Capture Agent Account has been configured') + elif not account['password']: + print(f"WARNING: No password configured for Capture Agent User {account['username']}") + # Check if account has api access + else: + __check_access(account=account, tenant_id=tenant_id) + + +def __check_access(account: dict, tenant_id: str) -> bool: + """ + Checks if the capture agent defined in the config has access to the ingest service. + The check tries to access the ingest service with the username and password defined in the config, + and sends a get request to '/services/available.json' . + If check fails, prints a warning. + :param account: The user defined in the config + :type account: dict + :param tenant_id: The target tenant + :type tenant_id: String + :return: bool + """ + log(f"Checking access for Capture Agent Account {account['username']}") + + url = f'{CONFIG.tenant_urls[tenant_id]}/services/available.json?serviceType=org.opencastproject.ingest' + login = { + 'user': account['username'], + 'password': account['password'] + } + try: + response = get_request(url, login, '/services/available.json?serviceType=org.opencastproject.ingest', + use_digest=False) + except RequestError: + print(f"WARNING: Capture Agent {account['username']} has no access.") + return False + except Exception as e: + print('ERROR: Failed to check for API access.') + print(str(e)) + return False + + if 'services' in response.json().keys(): + return True + else: + return False diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index ff2e167..1ae6fd7 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -6,20 +6,21 @@ from parsing_configurations import parse_args, read_yaml_file, parse_config from configure_users import check_users, set_config_users from configure_groups import check_groups, set_config_groups +from configure_capture_accounts import check_capture_accounts, set_config_capture_accounts import config def main(): ### Parse args and config ### - # ToDo Think about whether we should exclude Digest Login credentials from config.py file digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # create Digest Login environment, tenant_id, check = parse_args() # parse args env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file script_config = parse_config(config, env_conf, digest_login) # parse config.py group_config = read_yaml_file(script_config.group_path) # read group config file - set_config_users(digest_login, env_conf, script_config) # import config to the user script - set_config_groups(digest_login, group_config, script_config) # import config to the group script + set_config_users(digest_login, env_conf, script_config) # import config to user script + set_config_groups(digest_login, group_config, script_config) # import config to group script + set_config_capture_accounts(digest_login, env_conf, script_config) # import config to capture script # if tenant is not given, we perform the checks for all tenants if tenant_id: @@ -32,13 +33,13 @@ def main(): if check == 'all': check_users(tenant_id) check_groups(tenant_id) - # ToDo switchcast_system_accounts(tenant_id) + check_capture_accounts(tenant_id) elif check == 'users': check_users(tenant_id) elif check == 'groups': check_groups(tenant_id) - # elif check == 'capture': - # switchcast_system_accounts(tenant_id) + elif check == 'capture': + check_capture_accounts(tenant_id) if __name__ == '__main__': From 4512e5a0c4c9ae7dcabe098a5966eb2830950c0b Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 3 Aug 2021 23:16:27 +0200 Subject: [PATCH 46/79] Changed ID of opencast organizaion 'dummy' to 'all' --- multi-tenant-configuration/configure_users.py | 23 ++++--------------- .../staging/opencast-organizations.yml | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 8b15476..f4023cc 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -3,14 +3,13 @@ from args.digest_login import DigestLogin from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission -from parsing_configurations import __abort_script, log +from parsing_configurations import log CONFIG = None ENV_CONFIG = None DIGEST_LOGIN = None -# ToDo should this be moved to the config file? UNEXPECTED_ROLES = ["ROLE_ADMIN", "ROLE_ADMIN_UI", "ROLE_UI_", "ROLE_CAPTURE_"] @@ -45,12 +44,12 @@ def check_users(tenant_id: str): # Check and configure System User Accounts & External API User Accounts: for organization in ENV_CONFIG['opencast_organizations']: # check switchcast system accounts - if organization['id'] == 'dummy': + if organization['id'] == 'all': log(f'Checking system accounts for tenant {tenant_id} ...') for system_account in organization['switchcast_system_accounts']: __check_user(system_account, tenant_id) # check and configure external api accounts - if organization['id'] == tenant_id: # ToDo or 'all' ? + if organization['id'] == tenant_id: log(f'Checking External API accounts for tenant {tenant_id} ...') for user in organization['external_api_accounts']: __check_user(user, tenant_id) @@ -91,7 +90,7 @@ def __check_user(user: dict, tenant_id: str): __check_effective_user_roles(user, tenant_id) -def __check_api_access(user: dict, tenant_id: str) -> bool: +def __check_api_access(user: dict, tenant_id: str): """ Checks if the user defined in the config has access to the API. The check tries to login with the username and password defined in the config, @@ -101,7 +100,6 @@ def __check_api_access(user: dict, tenant_id: str) -> bool: :type user: Dict :param tenant_id: The target tenant :type tenant_id: String - :return: True """ log(f"Checking API access for user {user['username']}") @@ -115,7 +113,6 @@ def __check_api_access(user: dict, tenant_id: str) -> bool: get_request(url, login, '/api/info/me', headers=headers, use_digest=False) except RequestError: print(f"User {user['username']} has no API Access") - # ToDo add to group to get API access roles? action_allowed = check_or_ask_for_permission( target_type='user', action='configure user', @@ -125,11 +122,8 @@ def __check_api_access(user: dict, tenant_id: str) -> bool: if action_allowed: update_user(tenant_id, user=user) except Exception as e: - print('Error: Failed to check for API access.') + print('ERROR: Failed to check for API access.') print(str(e)) - return False - - return True def __check_effective_user_roles(user: dict, tenant_id: str): @@ -140,7 +134,6 @@ def __check_effective_user_roles(user: dict, tenant_id: str): :type user: Dict :param tenant_id: The ID of the target tenant :type tenant_id: String - :return: True """ log(f"Check effective user roles of user {user['username']}") @@ -150,8 +143,6 @@ def __check_effective_user_roles(user: dict, tenant_id: str): if unexpected_role in role: print(f"WARNING: Unexpected role found for User {user['username']}: {role}") - return True - def __check_user_roles(user: dict, existing_user: dict, tenant_id: str): """ @@ -163,7 +154,6 @@ def __check_user_roles(user: dict, existing_user: dict, tenant_id: str): :type: Dict :param tenant_id: The target tenant :type: String - :return: True """ log(f"Check user roles of user {user['username']}") @@ -219,8 +209,6 @@ def __check_user_roles(user: dict, existing_user: dict, tenant_id: str): # roles = ",".join(roles) update_user(tenant_id, user, overwrite_roles=roles) - return True - def get_user(username: str, tenant_id: str): """ @@ -365,7 +353,6 @@ def extract_internal_user_roles(user: dict, as_string=False): """ roles = [] for role in user['roles']: - # ToDo check if 'INTERNAL' is the right thing to use here if role['type'] == 'INTERNAL': roles.append(role['name']) if as_string: diff --git a/multi-tenant-configuration/environment/staging/opencast-organizations.yml b/multi-tenant-configuration/environment/staging/opencast-organizations.yml index 0714693..0ecf918 100644 --- a/multi-tenant-configuration/environment/staging/opencast-organizations.yml +++ b/multi-tenant-configuration/environment/staging/opencast-organizations.yml @@ -1,7 +1,7 @@ --- opencast_organizations: - - id: dummy + - id: all name: Dummy Tenant aai_org: switch.ch stream_sec_key: 5387689 From 9e1e6ceb5aa812e7290ecbff873177b803ee6de1 Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 3 Aug 2021 23:19:42 +0200 Subject: [PATCH 47/79] Adapted the group description in the configuration file to allow python formatting via .format() switched from ${name} to {tenant_id} --- .../configurations/group_configuration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-tenant-configuration/configurations/group_configuration.yaml b/multi-tenant-configuration/configurations/group_configuration.yaml index 52cd805..e37796c 100644 --- a/multi-tenant-configuration/configurations/group_configuration.yaml +++ b/multi-tenant-configuration/configurations/group_configuration.yaml @@ -23,7 +23,7 @@ groups: - ROLE_ADMIN - ROLE_SUDO - name: Organization Administrators - description: Organization administrators have full access to all content of ${name} + description: Organization administrators have full access to all content of {tenant_id} tenants: all type: open members: [] From b49fbbb85023a215c602eed8d07ade1c2c32f70f Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 3 Aug 2021 23:26:11 +0200 Subject: [PATCH 48/79] finalized group check - added warning when a group contains admin or sudo rights - removed unneccesary ToDos - changed group description template - changed group identifier - minor code improvements --- .../configure_groups.py | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index b143d07..2306739 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -55,8 +55,6 @@ def __check_group(group: dict, tenant_id: str): - checks if group description matches - checks if group members match - checks if group roles match - - ToDo Check API access of all group members - - ToDo Check group type :param group: The group to be checked :type group: dict :param tenant_id: The target tenant @@ -88,12 +86,6 @@ def __check_group(group: dict, tenant_id: str): # Check if group roles match the group roles provided in the configuration. # Update group roles if they do not match. (Asks for permission) __check_group_roles(group=group, existing_group=existing_group, tenant_id=tenant_id) - # ToDo - # Check external API accounts of members. Add missing API accounts. - # ToDo should this actually be done? We already ask if members should be removed in the member check. - # Check group type. If group is closed, remove unexpected members. - # Check for invalid group type - # Update group members. (Asks for permission) def __check_group_description(group: dict, existing_group: dict, tenant_id: str): @@ -109,11 +101,6 @@ def __check_group_description(group: dict, existing_group: dict, tenant_id: str) """ log(f"check names and description for group {group['name']}.") - # ToDo: does it really makes sense to check for the name? - # This seems to be already done when checking for the existence of the group. - if group['name'] != existing_group['name']: - print("WARNING: Group names do not match. ") - return if __group_description_template(group['description'], tenant_id) == existing_group['description']: log('Group descriptions match.') else: @@ -140,6 +127,10 @@ def __check_group_members(group: dict, existing_group: dict, tenant_id: str): """ log(f"Check members for group {group['name']}.") + group_roles = __extract_roles_from_group(group=group, tenant_id=tenant_id) + if "ROLE_ADMIN" or "ROLE_SUDO" in group_roles: + print("ATTENTION: This group contains admin or sudo rights!") + group_members = __extract_members_from_group(group=group, tenant_id=tenant_id) existing_group_members = sorted(filter(None, existing_group['members'].split(","))) log("Config group members: ", group_members) @@ -309,7 +300,7 @@ def create_group(group: dict, tenant_id: str): # check if member exist on tenant for member in members: if not get_user(username=member, tenant_id=tenant_id): - print(f"Warning: Member {member} does not exist.") + print(f"WARNING: Member {member} does not exist.") members.remove(member) members = ",".join(members) roles = __extract_roles_from_group(group, tenant_id, as_string=True) @@ -320,6 +311,7 @@ def create_group(group: dict, tenant_id: str): 'roles': roles, 'members': members, } + print(data) try: response = post_request(url, DIGEST_LOGIN, '/api/groups/', data=data) except RequestError as err: @@ -359,7 +351,6 @@ def update_group(tenant_id: str, group: dict, """ log(f"Trying to update group ... ") - group_id = group['identifier'] name = overwrite_name if overwrite_name else group['name'] description = overwrite_description if overwrite_description else \ __group_description_template(group['description'], tenant_id) @@ -368,7 +359,7 @@ def update_group(tenant_id: str, group: dict, members = overwrite_members if overwrite_members else \ __extract_members_from_group(group, tenant_id, as_string=True) - url = f'{CONFIG.tenant_urls[tenant_id]}/api/groups/{group_id}' + url = f"{CONFIG.tenant_urls[tenant_id]}/api/groups/{group['identifier']}" data = { 'name': name, 'description': description, @@ -378,7 +369,7 @@ def update_group(tenant_id: str, group: dict, try: response = put_request(url, DIGEST_LOGIN, '/api/groups/{groupId}', data=data) except RequestError as err: - if err.get_status_code() == "400": # ToDo: check if this is actually 404 + if err.get_status_code() == "404": print(f"Bad Request: Group with name {name} does not exist.") print("RequestError: ", err) return False @@ -399,9 +390,8 @@ def __generate_group_identifier(group: dict, tenant_id: str): :type tenant_id: str :return: The group id: str """ - # ToDo check if the generated identifiers are correct! (the same as in the ruby script) - # return f"{tenant_id}_{group['name'].replace(' ', '_')}".lower() - return group['name'].replace(' ', '_').lower() + identifier = group['name'].replace(' ', '_').lower() + return identifier def __group_description_template(description: str, tenant_id: str): @@ -413,9 +403,7 @@ def __group_description_template(description: str, tenant_id: str): :type tenant_id: str :return: group description with the inserted name, str """ - # ToDo check for a better way to insert into template - description = description.replace("${name}", tenant_id) - + description = description.format(tenant_id=tenant_id) return description From 1c337e208caa8666fb4eb61823d8ff16b0ce0a2a Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 3 Aug 2021 23:28:48 +0200 Subject: [PATCH 49/79] minor changes: added capture check, removed ToDos, and improved code --- .../parsing_configurations.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 6a73ba5..b4cc871 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -10,10 +10,9 @@ def parse_args(): """ - Parse the arguments and check them for correctness - - :return: - :rtype: + Parses the arguments and check them for correctness + :return: the environment, the tenant_id, the check + :rtype: triple """ parser, optional_args, required_args = get_args_parser() @@ -38,8 +37,8 @@ def parse_args(): if not args.check: args.check = ['all'] - elif args.check[0] not in ['users', 'groups', 'cast', 'capture']: - args_error(parser, "The check should be 'users', 'groups', 'cast' or 'capture'") + elif args.check[0] not in ['users', 'groups', 'capture']: + args_error(parser, "The check should be 'users', 'groups' or 'capture'") global VERBOSE_FLAG if args.verbose and args.verbose[0] == "True": @@ -56,8 +55,6 @@ def read_yaml_file(path): :param path: path to the yaml file :return: returns a dictionary """ - # ToDo error handling if path or file does not exist - # FileNotFoundError: with open(path, 'r') as f: content = yaml.load(f, Loader=yaml.FullLoader) @@ -65,9 +62,6 @@ def read_yaml_file(path): def parse_config(config, env_config, digest_login): - # ToDo Check if all mandatory configurations are given i.e. url pattern or a url for all tenants - - # ToDo should mh_default_org be removed from tenant_ids? config.tenant_ids = get_tenants(config.base_url, digest_login) config.tenant_ids.remove('mh_default_org') @@ -80,7 +74,7 @@ def parse_config(config, env_config, digest_login): return config -def create_group_config_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): +def create_yaml_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): """ This function can be used to transform a json file to a yaml file. requires import json and import yaml From 85766ec95aae9c7d1c3c12c1eda89c888455b722 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 12:48:24 +0200 Subject: [PATCH 50/79] updated doc string in get_request() --- lib/rest_requests/request.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/rest_requests/request.py b/lib/rest_requests/request.py index 9eb7883..d62e47c 100644 --- a/lib/rest_requests/request.py +++ b/lib/rest_requests/request.py @@ -13,13 +13,14 @@ def get_request(url, login, element_description, asset_type_description=None, asset_description=None, stream=False, headers=None, use_digest=True): """ - Make a get request to the given url with the given digest login. If the request fails with an error or a status - code != 200, a Request Error with the error message /status code and the given descriptions is thrown. + Make a get request to the given url with the given login credentials (Either Basic Auth or Digest Login). + If the request fails with an error or a status code != 200, a Request Error with the error message /status code + and the given descriptions is thrown. :param url: URL to make get request to :type url: str - :param digest_login: The login credentials for digest authentication - :type digest_login: DigestLogin + :param login: The login credentials (either HTTP Basic or digest authentication) + :type login: Login :param element_description: Element description in case of errors, e.g. 'event', 'series', 'tenants' :type element_description: str :param asset_type_description: Asset type description in case of errors, e.g. 'series', 'episode' @@ -30,6 +31,8 @@ def get_request(url, login, element_description, asset_type_description=None, as :type stream: bool :param headers: The headers to include in the request :type headers: dict + :param use_digest: Whether to use digest login + :type use_digest: bool :return: response :raise RequestError: """ From 9a8981f861d54c69e327db1599ca129c1e557229 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 14:20:25 +0200 Subject: [PATCH 51/79] Added BasicLogin Wraper for Basic Authentification --- lib/args/basic_login.py | 3 +++ lib/rest_requests/request.py | 2 +- .../configure_capture_accounts.py | 15 ++++----------- multi-tenant-configuration/configure_users.py | 7 +++---- multi-tenant-configuration/main.py | 2 +- 5 files changed, 12 insertions(+), 17 deletions(-) create mode 100644 lib/args/basic_login.py diff --git a/lib/args/basic_login.py b/lib/args/basic_login.py new file mode 100644 index 0000000..717a9a6 --- /dev/null +++ b/lib/args/basic_login.py @@ -0,0 +1,3 @@ +from collections import namedtuple + +BasicLogin = namedtuple('BasicLogin', ['user', 'password']) diff --git a/lib/rest_requests/request.py b/lib/rest_requests/request.py index d62e47c..f845960 100644 --- a/lib/rest_requests/request.py +++ b/lib/rest_requests/request.py @@ -42,7 +42,7 @@ def get_request(url, login, element_description, asset_type_description=None, as auth = HTTPDigestAuth(login.user, login.password) headers["X-Requested-Auth"] = "Digest" else: - auth = HTTPBasicAuth(login['user'], login['password']) + auth = HTTPBasicAuth(login.user, login.password) try: response = requests.get(url, auth=auth, headers=headers, stream=stream) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index 76e554f..f2a7246 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -1,30 +1,25 @@ from rest_requests.request import get_request from rest_requests.request_error import RequestError -from args.digest_login import DigestLogin +from args.basic_login import BasicLogin from parsing_configurations import log CONFIG = None ENV_CONFIG = None -DIGEST_LOGIN = None -def set_config_capture_accounts(digest_login: DigestLogin, env_conf: dict, config: dict): +def set_config_capture_accounts(env_conf: dict, config: dict): """ Sets/imports the global config variables. must be called before any checks can be performed. - :param digest_login: The digest login to be used - :type digest_login: DigestLogin :param env_conf: The environment configuration which specifies the user and system accounts :type env_conf: dict :param config: The script configuration :type config: dict """ - global DIGEST_LOGIN global ENV_CONFIG global CONFIG - DIGEST_LOGIN = digest_login ENV_CONFIG = env_conf CONFIG = config @@ -82,10 +77,8 @@ def __check_access(account: dict, tenant_id: str) -> bool: log(f"Checking access for Capture Agent Account {account['username']}") url = f'{CONFIG.tenant_urls[tenant_id]}/services/available.json?serviceType=org.opencastproject.ingest' - login = { - 'user': account['username'], - 'password': account['password'] - } + login = BasicLogin(user=account['username'], password=account['password']) + try: response = get_request(url, login, '/services/available.json?serviceType=org.opencastproject.ingest', use_digest=False) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index f4023cc..e499596 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -1,5 +1,6 @@ from rest_requests.request import get_request, post_request, put_request from rest_requests.request_error import RequestError +from args.basic_login import BasicLogin from args.digest_login import DigestLogin from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission @@ -105,10 +106,8 @@ def __check_api_access(user: dict, tenant_id: str): url = f'{CONFIG.tenant_urls[tenant_id]}/api/info/me' headers = {} - login = { - 'user': user['username'], - 'password': user['password'] - } + login = BasicLogin(user=user['username'], password=user['password']) + try: get_request(url, login, '/api/info/me', headers=headers, use_digest=False) except RequestError: diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 1ae6fd7..0a4182e 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -20,7 +20,7 @@ def main(): group_config = read_yaml_file(script_config.group_path) # read group config file set_config_users(digest_login, env_conf, script_config) # import config to user script set_config_groups(digest_login, group_config, script_config) # import config to group script - set_config_capture_accounts(digest_login, env_conf, script_config) # import config to capture script + set_config_capture_accounts(env_conf, script_config) # import config to capture script # if tenant is not given, we perform the checks for all tenants if tenant_id: From a851a27c64b1a09091d9dae03cd12afe24a1fca5 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 16:49:06 +0200 Subject: [PATCH 52/79] Changed command argument from 'tenantid' to 'tenant-id' --- multi-tenant-configuration/README.md | 2 +- multi-tenant-configuration/parsing_configurations.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 4d2a08e..d6c1523 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -43,7 +43,7 @@ The script can be called with the following command (all parameters in brackets | Param | Description | | :---: | :---------- | | `-e` / `--environment` | The environment where to find the configuration file (either `staging` or `production`) | -| `-t` / `--tenantid` | The id of the target tenant to be configured | +| `-t` / `--tenant-id` | The id of the target tenant to be configured | | `-c` / `--check` | checks to be performed (`users`, `groups`, `cast` or `capture`) (default: `all`) | | `-v` / `--verbose` | enables logging to be prompted if set to `True` | diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index b4cc871..923bee8 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -18,7 +18,7 @@ def parse_args(): required_args.add_argument("-e", "--environment", type=str, nargs='+', help="the environment (either 'staging' or 'production')") - optional_args.add_argument("-t", "--tenantid", type=str, nargs='+', help="target tenant id") + optional_args.add_argument("-t", "--tenant-id", type=str, nargs='+', help="target tenant id") optional_args.add_argument("-c", "--check", type=str, nargs='+', help="checks to be performed ('users', 'groups', 'cast' or 'capture') (default: all)") optional_args.add_argument("-v", "--verbose", type=str, nargs='+', help="enables more logging") @@ -32,8 +32,8 @@ def parse_args(): if len(args.environment) > 1: args_error(parser, "You can only provide one environment. Either 'staging' or 'production'") - if not args.tenantid: - args.tenantid = [''] + if not args.tenant_id: + args.tenant_id = [''] if not args.check: args.check = ['all'] @@ -46,7 +46,7 @@ def parse_args(): else: VERBOSE_FLAG = False - return args.environment[0], args.tenantid[0], args.check[0] + return args.environment[0], args.tenant_id[0], args.check[0] def read_yaml_file(path): From e971f738f1ba6eba3fe344c9228e7c510f2dba3f Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:03:57 +0200 Subject: [PATCH 53/79] Updated README --- multi-tenant-configuration/README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index d6c1523..164ba6f 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -1,9 +1,21 @@ # Multi-tenants User configuration scripts for Opencast -**ToDo** -This script ... +This script simplifies the process of multi-tenant configuration. +It allows to read in configurations of tenants and opencast organizations and checks if these configurations match the ones found on the respective opencast system. -Currently this script does not ... +The *configuration* file `opencast-organizations.yml` in the environment folder contains specifications for: + +- Opencast Organizations +- Switchcast System Accounts +- Capture Agent Accounts +- Tenants +- Users (and their Roles) + +the *configuration* file `scripts/config/group_configuration.json` contains specifications for the Groups: +- Group name +- Group description +- Tenant (on which this group should exist) +- Group members ## How to Use From f610d001210309c8b1bcc5fe148e692d63489a67 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:19:47 +0200 Subject: [PATCH 54/79] Updated config and README --- multi-tenant-configuration/README.md | 32 +++++++++++----------------- multi-tenant-configuration/config.py | 5 ++--- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 164ba6f..99384bb 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -23,28 +23,20 @@ the *configuration* file `scripts/config/group_configuration.json` contains spec The script is configured by editing the values in `config.py`: -| Configuration Key | Description | Default/Example | -| :---------------- | :---------------------------------------- | :--------------------------- | -| `url` | The URL of the global admin node ? | https://tenant1.opencast.com | -| `tenant_url_pattern` | The URL pattern of the target tenants | https://tenant2.opencast.com | -| `tenant_urls` | A dictioanry of server URLs of the target tenants | https://tenant2.opencast.com | -| `digest_user` | The user name of the digest user | `opencast_system_account` | -| `digest_pw` | The password of the digest user | `CHANGE_ME` | -| `env_path` | The id of the workflow to start on ingest | reimport-workflow | - -**TODo**: check the below ... - -_The configured digest user needs to exist on both tenants and have the same password for both of them. This is because -the script ingests the assets via URL, which is faster, but the user needs to be able to access the source tenant from -the target tenant for this to work. Additionally the user currently needs to have ROLE_ADMIN to be able to use -`/assets/{episodeid}`._ - -_For the future, Basic Authentication and the use of an endpoint that doesn't require the Admin role (e.g. -`api/events/{id}`) would be preferable, so you can simply add a frontend user with the necessary rights (ingest, -access to the events/series) and the same password to both tenants._ +| Configuration Key | Description | Default/Example | +| :-------------------- | :-------------------------------------------- | :--------------------------- | +| `base_url` | The URL of the global admin node | `"http://localhost:8080"` | +| `tenant_url_pattern` | The URL pattern of the target tenants | `"http://{}:8080"` | +| `tenant_urls` | Optional dictionary of server URLs per tenant | `{'tenant1': 'http://tenant1:8080', 'tenant2': 'http://tenant2:8080'}` | +| `digest_user` | The user name of the digest user | `opencast_system_account` | +| `digest_pw` | The password of the digest user | `CHANGE_ME` | +| `env_path` | The path to the environment configuration file| `"environment/{}/opencast-organizations.yml"` | +| `group_path` | The path to the group configuration file | `"configurations/group_configuration.yaml"` | + +The configured digest user needs to exist on all tenants and has to have the same password. #### group config: -The names in the group config file must be unique per Tenant! +The group names in the group config file must be unique per Tenant! ### Usage diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 5989940..2ab0b27 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -1,6 +1,6 @@ # Configuration -# Set this to your global admin node +# Set this to your admin node base_url = "http://localhost:8080" # If you have multiple tenants, use an URL pattern. @@ -9,12 +9,11 @@ tenant_url_pattern = "http://{}:8080" # You can also define a dictionary of tenant URLs, which will be prioritized over the URL pattern: -# # examples: +# example: # tenant_urls = { # 'tenant1': 'http://tenant1:8080', # 'tenant2': 'http://tenant2:8080' # } -# tenant_urls = {'tenant1': 'https://develop.opencast.org'} # Digest User login digest_user = "opencast_system_account" From 3659eb806d674b8ccce56e48a6f1bcd4688a2c85 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:26:11 +0200 Subject: [PATCH 55/79] Refactored variable: base_url -> server_url --- multi-tenant-configuration/README.md | 2 +- multi-tenant-configuration/config.py | 2 +- multi-tenant-configuration/parsing_configurations.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 99384bb..6de9f76 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -25,7 +25,7 @@ The script is configured by editing the values in `config.py`: | Configuration Key | Description | Default/Example | | :-------------------- | :-------------------------------------------- | :--------------------------- | -| `base_url` | The URL of the global admin node | `"http://localhost:8080"` | +| `server_url` | The URL of the global admin node | `"http://localhost:8080"` | | `tenant_url_pattern` | The URL pattern of the target tenants | `"http://{}:8080"` | | `tenant_urls` | Optional dictionary of server URLs per tenant | `{'tenant1': 'http://tenant1:8080', 'tenant2': 'http://tenant2:8080'}` | | `digest_user` | The user name of the digest user | `opencast_system_account` | diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 2ab0b27..865da0d 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -1,7 +1,7 @@ # Configuration # Set this to your admin node -base_url = "http://localhost:8080" +server_url = "http://localhost:8080" # If you have multiple tenants, use an URL pattern. # example: diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 923bee8..e1ca45d 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -62,7 +62,7 @@ def read_yaml_file(path): def parse_config(config, env_config, digest_login): - config.tenant_ids = get_tenants(config.base_url, digest_login) + config.tenant_ids = get_tenants(config.server_url, digest_login) config.tenant_ids.remove('mh_default_org') if not hasattr(config, 'tenant_urls'): From 96e7636e2e2f7689187cc20200193146a435ec69 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:30:17 +0200 Subject: [PATCH 56/79] Updated README: Added explanation for optional tenant_urls dictionary --- multi-tenant-configuration/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 6de9f76..8284b68 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -35,6 +35,8 @@ The script is configured by editing the values in `config.py`: The configured digest user needs to exist on all tenants and has to have the same password. +The optional dictionary `tenant_urls` can be used if the tenant-id is not an exact part of the tenant URL or the URLs don't follow a common pattern. + #### group config: The group names in the group config file must be unique per Tenant! From 2446129220a73dd8f4862d0cb6f1ade97bff4762 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:43:32 +0200 Subject: [PATCH 57/79] Added explanation in config --- multi-tenant-configuration/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index 865da0d..aea731e 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -3,7 +3,7 @@ # Set this to your admin node server_url = "http://localhost:8080" -# If you have multiple tenants, use an URL pattern. +# If you have multiple tenants, use an URL pattern. The blank {} will be filled with the tenant-id. # example: # tenant_url_pattern = "https://{}.example.org" tenant_url_pattern = "http://{}:8080" From c32d02637121e3cb989415891e51edacc796b04c Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:49:33 +0200 Subject: [PATCH 58/79] Renamed config variables --- multi-tenant-configuration/README.md | 4 ++-- multi-tenant-configuration/config.py | 4 ++-- multi-tenant-configuration/main.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 8284b68..77f9082 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -30,8 +30,8 @@ The script is configured by editing the values in `config.py`: | `tenant_urls` | Optional dictionary of server URLs per tenant | `{'tenant1': 'http://tenant1:8080', 'tenant2': 'http://tenant2:8080'}` | | `digest_user` | The user name of the digest user | `opencast_system_account` | | `digest_pw` | The password of the digest user | `CHANGE_ME` | -| `env_path` | The path to the environment configuration file| `"environment/{}/opencast-organizations.yml"` | -| `group_path` | The path to the group configuration file | `"configurations/group_configuration.yaml"` | +| `org_config_path` | The path to the organization config file | `"environment/{}/opencast-organizations.yml"` | +| `group_config_path` | The path to the group config file | `"configurations/group_configuration.yaml"` | The configured digest user needs to exist on all tenants and has to have the same password. diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index aea731e..d0ef91b 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -20,6 +20,6 @@ digest_pw = "CHANGE_ME" # path to environment configuration file -env_path = "environment/{}/opencast-organizations.yml" +org_config_path = "environment/{}/opencast-organizations.yml" # path to group configuration file -group_path = "configurations/group_configuration.yaml" +group_config_path = "configurations/group_configuration.yaml" diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 0a4182e..2f20922 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -15,9 +15,9 @@ def main(): ### Parse args and config ### digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # create Digest Login environment, tenant_id, check = parse_args() # parse args - env_conf = read_yaml_file(config.env_path.format(environment)) # read environment config file + env_conf = read_yaml_file(config.org_config_path.format(environment)) # read environment config file script_config = parse_config(config, env_conf, digest_login) # parse config.py - group_config = read_yaml_file(script_config.group_path) # read group config file + group_config = read_yaml_file(script_config.group_config_path) # read group config file set_config_users(digest_login, env_conf, script_config) # import config to user script set_config_groups(digest_login, group_config, script_config) # import config to group script set_config_capture_accounts(env_conf, script_config) # import config to capture script From 646f6e7833ffd520ac9c4f0598be5ef91ea7a826 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 17:53:02 +0200 Subject: [PATCH 59/79] Updated config: Added explanation --- multi-tenant-configuration/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index d0ef91b..b3124b2 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -19,7 +19,8 @@ digest_user = "opencast_system_account" digest_pw = "CHANGE_ME" -# path to environment configuration file +# path to environment configuration file. +# The {} are a placeholder which will be filled with the environment passed as an argument (e.g. staging or production). org_config_path = "environment/{}/opencast-organizations.yml" # path to group configuration file group_config_path = "configurations/group_configuration.yaml" From a3cf83de091f0e9d4caf94941cc16a252307f7f6 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:00:34 +0200 Subject: [PATCH 60/79] fixed json --- .../configurations/group_configuration.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-tenant-configuration/configurations/group_configuration.json b/multi-tenant-configuration/configurations/group_configuration.json index a27ceb1..47510fe 100644 --- a/multi-tenant-configuration/configurations/group_configuration.json +++ b/multi-tenant-configuration/configurations/group_configuration.json @@ -68,7 +68,7 @@ "tenants": "all", "roles": [ "ROLE_ADMIN_UI", - "ROLE_UI_EVENTS_CREATE" + "ROLE_UI_EVENTS_CREATE" ] }, { From 7f7f517b620bed262fc5a0e2dbf0c7b90f9f8ad2 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:02:19 +0200 Subject: [PATCH 61/79] Removed group types The group types (closed or open) are not used in the script anymore. The were formerly used to allow certain members to be added without asking the user again. Now, as user permissions can be stored, this is not neccesary anymore. The user is asked for every member, if no permissions has been specified before. --- .../configurations/group_configuration.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/multi-tenant-configuration/configurations/group_configuration.yaml b/multi-tenant-configuration/configurations/group_configuration.yaml index e37796c..58b1cfd 100644 --- a/multi-tenant-configuration/configurations/group_configuration.yaml +++ b/multi-tenant-configuration/configurations/group_configuration.yaml @@ -4,7 +4,6 @@ groups: - name: System Administrators description: System Administrators tenants: all - type: closed members: - name: Guy 1 email: test@test.de @@ -25,7 +24,6 @@ groups: - name: Organization Administrators description: Organization administrators have full access to all content of {tenant_id} tenants: all - type: open members: [] inactive_members: [] permissions: @@ -42,7 +40,6 @@ groups: - name: Producers description: Producers have limited access to content and functionality tenants: all - type: open members: [] inactive_members: [] permissions: @@ -64,7 +61,6 @@ groups: - name: Tenant1 Producers description: Tenant1 Producers have limited access to content and functionality tenants: tenant1 - type: open members: - name: Guy X email: test@test.de From 82f33084bd90f732e89a72027457ff0d3b47830c Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:15:09 +0200 Subject: [PATCH 62/79] changed uid to username in yaml file and script --- .../configurations/group_configuration.yaml | 8 ++++---- multi-tenant-configuration/configure_groups.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/multi-tenant-configuration/configurations/group_configuration.yaml b/multi-tenant-configuration/configurations/group_configuration.yaml index 58b1cfd..c3e4b35 100644 --- a/multi-tenant-configuration/configurations/group_configuration.yaml +++ b/multi-tenant-configuration/configurations/group_configuration.yaml @@ -8,12 +8,12 @@ groups: - name: Guy 1 email: test@test.de reason: Operations partner - uid: guy1 + username: guy1 tenants: all - name: Guy 2 email: test@test.de reason: Operations partner - uid: guy2 + username: guy2 tenants: tenant1 inactive_members: [] permissions: @@ -65,12 +65,12 @@ groups: - name: Guy X email: test@test.de reason: Operations partner - uid: guyx + username: guyx tenants: all - name: Guy 2 email: test@test.de reason: Operations partner - uid: guy2 + username: guy2 tenants: tenant1 inactive_members: [] permissions: diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 2306739..689e57c 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -419,7 +419,7 @@ def __extract_members_from_group(group: dict, tenant_id: str, as_string=False): :type as_string: bool :return: Comma separated string of members (e.g. "guy1,guy2") or list of members. """ - members = [member['uid'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] + members = [member['username'] for member in group['members'] if member['tenants'] in ['all', tenant_id]] if as_string: members = ",".join(sorted(members)) return members From b044742c32110b96b42c41f1eec2452253a62369 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:18:40 +0200 Subject: [PATCH 63/79] Updated README: Added explanation group configplaceholder --- multi-tenant-configuration/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 77f9082..221dc49 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -40,6 +40,8 @@ The optional dictionary `tenant_urls` can be used if the tenant-id is not an exa #### group config: The group names in the group config file must be unique per Tenant! +In the group description, python placeholder can be used (i.e. `{tenant_id}`) to include the current tenant-id in the description. + ### Usage The script can be called with the following command (all parameters in brackets are optional): From c1bdb2cf8a678957f6769e6d88e48f1994a419db Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:19:36 +0200 Subject: [PATCH 64/79] removed blank line --- multi-tenant-configuration/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/multi-tenant-configuration/requirements.txt b/multi-tenant-configuration/requirements.txt index ba9cec7..a4e02d8 100644 --- a/multi-tenant-configuration/requirements.txt +++ b/multi-tenant-configuration/requirements.txt @@ -4,5 +4,4 @@ idna==2.10 requests==2.25.0 requests-toolbelt==0.9.1 urllib3==1.26.2 - pyyaml==5.3.1 From f4ca5ffdd8ddd8e745c3bb5b24983b8b32b178b7 Mon Sep 17 00:00:00 2001 From: mheyen Date: Sun, 29 Aug 2021 18:32:21 +0200 Subject: [PATCH 65/79] Changed organization config file removed anything which is not used by the script. --- multi-tenant-configuration/configure_users.py | 2 +- .../staging/opencast-organizations.yml | 16 +--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index e499596..997d744 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -45,7 +45,7 @@ def check_users(tenant_id: str): # Check and configure System User Accounts & External API User Accounts: for organization in ENV_CONFIG['opencast_organizations']: # check switchcast system accounts - if organization['id'] == 'all': + if organization['id'] == 'All Tenants': log(f'Checking system accounts for tenant {tenant_id} ...') for system_account in organization['switchcast_system_accounts']: __check_user(system_account, tenant_id) diff --git a/multi-tenant-configuration/environment/staging/opencast-organizations.yml b/multi-tenant-configuration/environment/staging/opencast-organizations.yml index 0ecf918..d4a35a6 100644 --- a/multi-tenant-configuration/environment/staging/opencast-organizations.yml +++ b/multi-tenant-configuration/environment/staging/opencast-organizations.yml @@ -1,18 +1,8 @@ --- opencast_organizations: - - id: all + - id: All Tenants name: Dummy Tenant - aai_org: switch.ch - stream_sec_key: 5387689 - acl_default_template: organization - acl_default_download: False - acl_default_annotate: False - - # Global External API user passwords - opencast_system_account: - username: opencast_system_account - password: CHANGE_ME switchcast_system_accounts: - username: player name: Player System User @@ -33,8 +23,6 @@ opencast_organizations: - id: tenant1 name: Tenant1 - aai_org: tenant1.ch - stream_sec_key: tu7uzgjjhghjf capture_agent_accounts: - username: ca-tenant1-ch password: jvblkajklvjhaklehr @@ -56,8 +44,6 @@ opencast_organizations: roles: [ROLE_ADMIN, ROLE_SUDO] - id: tenant2 name: Tenant2 - aai_org: tenant2.ch - stream_sec_key: tu7uzgjjhghjf capture_agent_accounts: - username: ca-tenant2-ch password: hjfkhfzuruzf76 From 0e11c1af5dda30858c69d08eb466d641b672f744 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 19:50:47 +0200 Subject: [PATCH 66/79] Added doc strings and moved private functions --- .../user_interaction.py | 117 +++++++++++++++--- 1 file changed, 97 insertions(+), 20 deletions(-) diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py index 1027413..6e2cfc8 100644 --- a/multi-tenant-configuration/user_interaction.py +++ b/multi-tenant-configuration/user_interaction.py @@ -11,6 +11,21 @@ def check_or_ask_for_permission(target_type, action, target_name, tenant_id, option_i=False) -> bool: + """ + Check if a permission for the action was already given or asks for permission. + + :param target_type: The target for the action (either 'user' or 'group') + :type target_type: str + :param action: The action to be performed + :type action: str + :param target_name: The group or user name + :type target_name: str + :param tenant_id: The target tenant + :type tenant_id: str + :param option_i: Whether the user has the option to perform the action iteratively. + :type option_i: bool + :return: bool, whether the action should be performed. + """ # check if permission is already defined permission = get_permission(target_type, action, target_name, tenant_id) @@ -24,6 +39,20 @@ def check_or_ask_for_permission(target_type, action, target_name, tenant_id, opt def get_permission(target_type, action, target_name, tenant_id) -> bool: + """ + Returns the permission for the given action. + If no permission value is found, None is returned + + :param target_type: The target for the action (either 'user' or 'group') + :type target_type: str + :param action: The action to be performed + :type action: str + :param target_name: The group or user name + :type target_name: str + :param tenant_id: The target tenant + :type tenant_id: str + :return: bool or None, the permission value + """ log('permissions: ', permissions) @@ -44,13 +73,36 @@ def get_permission(target_type, action, target_name, tenant_id) -> bool: def ask_user(target_type, action, target_name, tenant_id, option_i=False) -> str: - - individual_option = "\n Write 'i' to perform the action individually for each case. " if option_i else "" - - help_description = f""" Write 'y' to perform the action. Write 'n' to skipp this action. {individual_option} - Add 't' for 'tenant' or 'a' for 'all' to store your decision for this or all tenants. - Add 't' for 'target' or 'a' for 'all' to store your decision for this or all targets. - EXAMPLE: Write 'yat' to store your decision for the action on ALL tenants and for THIS target. + """ + Asks the user for permission to perform a certain action. + Returns the answer. + The answer can be stored if the user specifies if the answer holds for: + - all tenants, or + - all targets (i.e. all groups or all ), or + - both. + This can done by adding 'a' or 't' to the answer. + For example: + 'yat' corresponds to 'Yes, always do this action on all tenants for this target'. + + :param target_type: The target for the action (either 'user' or 'group') + :type target_type: str + :param action: The action in question + :type action: str + :param target_name: The group or user name + :type target_name: str + :param tenant_id: The target tenant + :type tenant_id: str + :param option_i: Whether the user has the option to answer with i. + :type option_i: bool + :return: str, the answer of the user + """ + + individual_option = "\nWrite 'i' to perform the action individually for each case. " if option_i else "" + + help_description = f"""Write 'y' to perform the action. Write 'n' to skipp this action. {individual_option} +Add 't' for 'tenant' or 'a' for 'all' to store your decision for this or all tenants. +Add 't' for 'target' or 'a' for 'all' to store your decision for this or all targets. +EXAMPLE: Write 'yat' to store your decision for the action on ALL tenants and for THIS target. """ question = f"Do you want to {action} ({target_type} {target_name} on {tenant_id})? Write '{HELP_OPTION}' for help.\n" @@ -62,26 +114,30 @@ def ask_user(target_type, action, target_name, tenant_id, option_i=False) -> str if answer == HELP_OPTION: answer = input(help_description) # return all valid answers - elif parsable(answer) or (option_i and answer == 'i'): + elif __parsable(answer, option_i): return answer # catch all invalid answers else: answer = input(f"Invalid answer. Write '{HELP_OPTION}' for help.\n").lower() -def parsable(answer) -> bool: - - if re.match(ANSWER_PATTERN, answer) or answer == HELP_OPTION: - return True - else: - return False - - -def __build_key(action, tenant, target): - return action + ':' + tenant + ':' + target - - def process_answer(answer, target_type, action, target_name, tenant_id, option_i) -> bool: + """ + Processes an answer and, if specified, stores it as a permission. + Returns a boolean, whether the action should be performed. + + :param target_type: The target for the action (either 'user' or 'group') + :type target_type: str + :param action: The action to be performed + :type action: str + :param target_name: The group or user name + :type target_name: str + :param tenant_id: The target tenant + :type tenant_id: str + :param option_i: Whether the user has the option to perform the action iteratively. + :type option_i: bool + :return: bool, whether the action should be performed. + """ # simple yes or no case (not stored) if answer == 'y': @@ -100,3 +156,24 @@ def process_answer(answer, target_type, action, target_name, tenant_id, option_i permissions[target_type][key] = permission_value return permission_value + + +def __parsable(answer, option_i=False) -> bool: + """ + Checks if an answer is parsable, i.e. matches the answer pattern. + + :param answer: The answer given by the user + :type answer: str + :param option_i: Whether 'i' is an acceptable answer + :type option_i: str + :return: bool, whether the answer is parsable + """ + + return re.match(ANSWER_PATTERN, answer) or answer == HELP_OPTION or (option_i and answer == 'i') + + +def __build_key(action, tenant, target): + """ + Builds the key to store the permission in the dictionary. + """ + return action + ':' + tenant + ':' + target From cde69f78cb0925d35ea74c3dcb8b5cebe8bf3344 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 20:21:49 +0200 Subject: [PATCH 67/79] moved generic functions to lib folder --- lib/input_output/yaml_utils.py | 31 +++++++++++++++++++ multi-tenant-configuration/main.py | 3 +- .../parsing_configurations.py | 31 ------------------- 3 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 lib/input_output/yaml_utils.py diff --git a/lib/input_output/yaml_utils.py b/lib/input_output/yaml_utils.py new file mode 100644 index 0000000..d8d1726 --- /dev/null +++ b/lib/input_output/yaml_utils.py @@ -0,0 +1,31 @@ +import yaml +import json + + +def read_yaml_file(path): + """ + reads a .yaml file and returns a dictionary + :param path: path to the yaml file + :return: returns a dictionary + """ + with open(path, 'r') as f: + content = yaml.load(f, Loader=yaml.FullLoader) + + return content + + +def create_yaml_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): + """ + This function can be used to transform a json file to a yaml file. + requires import json and import yaml + :param json_file_path: path to json file + :param yaml_file_path: path to yaml file (will be created if it does not exist) + :return: + """ + + with open(json_file_path, 'r') as json_file: + json_data = json.load(json_file) + with open(yaml_file_path, 'w') as file: + yaml.dump(json_data, file, sort_keys=False) + + return True diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 2f20922..471e1e0 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -3,7 +3,8 @@ sys.path.append(os.path.join(os.path.abspath('..'), "lib")) from args.digest_login import DigestLogin -from parsing_configurations import parse_args, read_yaml_file, parse_config +from input_output.yaml_utils import read_yaml_file +from parsing_configurations import parse_args, parse_config from configure_users import check_users, set_config_users from configure_groups import check_groups, set_config_groups from configure_capture_accounts import check_capture_accounts, set_config_capture_accounts diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index e1ca45d..e10f86a 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -1,5 +1,3 @@ -import yaml -import json from args.args_parser import get_args_parser from args.args_error import args_error from rest_requests.basic_requests import get_tenants @@ -49,18 +47,6 @@ def parse_args(): return args.environment[0], args.tenant_id[0], args.check[0] -def read_yaml_file(path): - """ - reads a .yaml file and returns a dictionary - :param path: path to the yaml file - :return: returns a dictionary - """ - with open(path, 'r') as f: - content = yaml.load(f, Loader=yaml.FullLoader) - - return content - - def parse_config(config, env_config, digest_login): config.tenant_ids = get_tenants(config.server_url, digest_login) config.tenant_ids.remove('mh_default_org') @@ -74,23 +60,6 @@ def parse_config(config, env_config, digest_login): return config -def create_yaml_file_from_json_file(json_file_path, yaml_file_path='test.yaml'): - """ - This function can be used to transform a json file to a yaml file. - requires import json and import yaml - :param json_file_path: path to json file - :param yaml_file_path: path to yaml file (will be created if it does not exist) - :return: - """ - - with open(json_file_path, 'r') as json_file: - json_data = json.load(json_file) - with open(yaml_file_path, 'w') as file: - yaml.dump(json_data, file, sort_keys=False) - - return True - - def log(*args): if VERBOSE_FLAG: print(*args) From 0b00ec923034bb221a9d516659d771bdf873e4ea Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 20:22:49 +0200 Subject: [PATCH 68/79] Removed condition for verbose flag to be set to True --- multi-tenant-configuration/parsing_configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index e10f86a..522acae 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -39,7 +39,7 @@ def parse_args(): args_error(parser, "The check should be 'users', 'groups' or 'capture'") global VERBOSE_FLAG - if args.verbose and args.verbose[0] == "True": + if args.verbose: VERBOSE_FLAG = True else: VERBOSE_FLAG = False From c360adff8218a72716474dfc249a9f8d3fa0b8ad Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 20:28:06 +0200 Subject: [PATCH 69/79] removed abort_script() function --- multi-tenant-configuration/parsing_configurations.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index 522acae..eea37a4 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -63,8 +63,3 @@ def parse_config(config, env_config, digest_login): def log(*args): if VERBOSE_FLAG: print(*args) - - -def __abort_script(message): - print(message) - sys.exit() From f484f779dc2e17f6c9f410b2ec187b15b5392aa4 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 22:04:28 +0200 Subject: [PATCH 70/79] Restructured parsing_configuration.py and added Logger class Moved or deleted functions from parsing_configuration.py Created minimal Logger class and included the logger in all scripts --- lib/input_output/logger.py | 16 +++++++++ .../configure_capture_accounts.py | 8 +++-- .../configure_groups.py | 8 +++-- multi-tenant-configuration/configure_users.py | 8 +++-- multi-tenant-configuration/main.py | 35 ++++++++++++------- .../parsing_configurations.py | 30 ++-------------- .../user_interaction.py | 3 -- 7 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 lib/input_output/logger.py diff --git a/lib/input_output/logger.py b/lib/input_output/logger.py new file mode 100644 index 0000000..6f53a64 --- /dev/null +++ b/lib/input_output/logger.py @@ -0,0 +1,16 @@ + +class Logger: + """ + minimal logger to either be verbose or don't print messages at all. + """ + verbose = True + + def __init__(self, verbose: bool): + """ + Constructor + """ + self.verbose = verbose + + def log(self, *args): + if self.verbose: + print(*args) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index f2a7246..2fe18c3 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -1,14 +1,14 @@ from rest_requests.request import get_request from rest_requests.request_error import RequestError from args.basic_login import BasicLogin -from parsing_configurations import log +from input_output.logger import Logger CONFIG = None ENV_CONFIG = None -def set_config_capture_accounts(env_conf: dict, config: dict): +def set_config_capture_accounts(env_conf: dict, config: dict, logger: Logger): """ Sets/imports the global config variables. must be called before any checks can be performed. @@ -16,12 +16,16 @@ def set_config_capture_accounts(env_conf: dict, config: dict): :type env_conf: dict :param config: The script configuration :type config: dict + :param logger: A Logger instance + :type logger: Logger """ global ENV_CONFIG global CONFIG + global log ENV_CONFIG = env_conf CONFIG = config + log = logger.log def check_capture_accounts(tenant_id: str): diff --git a/multi-tenant-configuration/configure_groups.py b/multi-tenant-configuration/configure_groups.py index 689e57c..b41f17e 100644 --- a/multi-tenant-configuration/configure_groups.py +++ b/multi-tenant-configuration/configure_groups.py @@ -2,9 +2,9 @@ from rest_requests.request_error import RequestError from args.digest_login import DigestLogin from configure_users import get_user +from input_output.logger import Logger from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission -from parsing_configurations import log CONFIG = None @@ -12,7 +12,7 @@ DIGEST_LOGIN = None -def set_config_groups(digest_login: DigestLogin, group_config: dict, config: dict): +def set_config_groups(digest_login: DigestLogin, group_config: dict, config: dict, logger: Logger): """ Sets/imports the global config variables. Must be called before any checks can be performed. @@ -22,14 +22,18 @@ def set_config_groups(digest_login: DigestLogin, group_config: dict, config: dic :type group_config: dict :param config: The script configuration :type config: dict + :param logger: A Logger instance + :type logger: Logger """ global DIGEST_LOGIN global GROUP_CONFIG global CONFIG + global log DIGEST_LOGIN = digest_login GROUP_CONFIG = group_config CONFIG = config + log = logger.log def check_groups(tenant_id: str): diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index 997d744..a8b8f9c 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -2,9 +2,9 @@ from rest_requests.request_error import RequestError from args.basic_login import BasicLogin from args.digest_login import DigestLogin +from input_output.logger import Logger from input_output.input import get_yes_no_answer from user_interaction import check_or_ask_for_permission -from parsing_configurations import log CONFIG = None @@ -14,7 +14,7 @@ UNEXPECTED_ROLES = ["ROLE_ADMIN", "ROLE_ADMIN_UI", "ROLE_UI_", "ROLE_CAPTURE_"] -def set_config_users(digest_login: DigestLogin, env_conf: dict, config: dict): +def set_config_users(digest_login: DigestLogin, env_conf: dict, config: dict, logger: Logger): """ Sets/imports the global config variables. must be called before any checks can be performed. @@ -24,14 +24,18 @@ def set_config_users(digest_login: DigestLogin, env_conf: dict, config: dict): :type env_conf: dict :param config: The script configuration :type config: dict + :param logger: A Logger instance + :type logger: Logger """ global DIGEST_LOGIN global ENV_CONFIG global CONFIG + global log DIGEST_LOGIN = digest_login ENV_CONFIG = env_conf CONFIG = config + log = logger.log def check_users(tenant_id: str): diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 471e1e0..12b4470 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -3,8 +3,10 @@ sys.path.append(os.path.join(os.path.abspath('..'), "lib")) from args.digest_login import DigestLogin +from input_output.logger import Logger from input_output.yaml_utils import read_yaml_file -from parsing_configurations import parse_args, parse_config +from rest_requests.basic_requests import get_tenants +from parsing_configurations import parse_args from configure_users import check_users, set_config_users from configure_groups import check_groups, set_config_groups from configure_capture_accounts import check_capture_accounts, set_config_capture_accounts @@ -14,20 +16,27 @@ def main(): ### Parse args and config ### - digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) # create Digest Login - environment, tenant_id, check = parse_args() # parse args - env_conf = read_yaml_file(config.org_config_path.format(environment)) # read environment config file - script_config = parse_config(config, env_conf, digest_login) # parse config.py - group_config = read_yaml_file(script_config.group_config_path) # read group config file - set_config_users(digest_login, env_conf, script_config) # import config to user script - set_config_groups(digest_login, group_config, script_config) # import config to group script - set_config_capture_accounts(env_conf, script_config) # import config to capture script + digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) + environment, tenant_id, check, verbose = parse_args() + logger = Logger(verbose) + # read and parse organization config + org_conf = read_yaml_file(config.org_config_path.format(environment)) + config.tenant_ids = get_tenants(config.server_url, digest_login) + config.tenant_ids.remove('mh_default_org') + if not hasattr(config, 'tenant_urls'): + config.tenant_urls = {} + for tenant_id in config.tenant_ids: + if not tenant_id in config.tenant_urls: + config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) + # read group config + group_config = read_yaml_file(config.group_config_path) + # import config to scripts + set_config_users(digest_login, org_conf, config, logger) + set_config_groups(digest_login, group_config, config, logger) + set_config_capture_accounts(org_conf, config, logger) # if tenant is not given, we perform the checks for all tenants - if tenant_id: - tenants_to_check = [tenant_id] - else: - tenants_to_check = script_config.tenant_ids + tenants_to_check = [tenant_id] if tenant_id else config.tenant_ids ### Start checks ### for tenant_id in tenants_to_check: diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parsing_configurations.py index eea37a4..ac2caa5 100644 --- a/multi-tenant-configuration/parsing_configurations.py +++ b/multi-tenant-configuration/parsing_configurations.py @@ -1,9 +1,5 @@ from args.args_parser import get_args_parser from args.args_error import args_error -from rest_requests.basic_requests import get_tenants - - -VERBOSE_FLAG = True def parse_args(): @@ -38,28 +34,6 @@ def parse_args(): elif args.check[0] not in ['users', 'groups', 'capture']: args_error(parser, "The check should be 'users', 'groups' or 'capture'") - global VERBOSE_FLAG - if args.verbose: - VERBOSE_FLAG = True - else: - VERBOSE_FLAG = False - - return args.environment[0], args.tenant_id[0], args.check[0] - - -def parse_config(config, env_config, digest_login): - config.tenant_ids = get_tenants(config.server_url, digest_login) - config.tenant_ids.remove('mh_default_org') - - if not hasattr(config, 'tenant_urls'): - config.tenant_urls = {} - for tenant_id in config.tenant_ids: - if not tenant_id in config.tenant_urls: - config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) - - return config - + verbose = True if args.verbose else False -def log(*args): - if VERBOSE_FLAG: - print(*args) + return args.environment[0], args.tenant_id[0], args.check[0], verbose diff --git a/multi-tenant-configuration/user_interaction.py b/multi-tenant-configuration/user_interaction.py index 6e2cfc8..0165324 100644 --- a/multi-tenant-configuration/user_interaction.py +++ b/multi-tenant-configuration/user_interaction.py @@ -1,4 +1,3 @@ -from parsing_configurations import log import re @@ -54,8 +53,6 @@ def get_permission(target_type, action, target_name, tenant_id) -> bool: :return: bool or None, the permission value """ - log('permissions: ', permissions) - key = __build_key(action, tenant_id, target_name) if key in permissions[target_type].keys(): return permissions[target_type][key] From c314f95bbcdf4ca47c39c4ca71be94317e2c051b Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 22:09:49 +0200 Subject: [PATCH 71/79] renamed file to parse_arguments.py --- multi-tenant-configuration/main.py | 2 +- .../{parsing_configurations.py => parse_arguments.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename multi-tenant-configuration/{parsing_configurations.py => parse_arguments.py} (100%) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index 12b4470..b5d5828 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -6,7 +6,7 @@ from input_output.logger import Logger from input_output.yaml_utils import read_yaml_file from rest_requests.basic_requests import get_tenants -from parsing_configurations import parse_args +from parse_arguments import parse_args from configure_users import check_users, set_config_users from configure_groups import check_groups, set_config_groups from configure_capture_accounts import check_capture_accounts, set_config_capture_accounts diff --git a/multi-tenant-configuration/parsing_configurations.py b/multi-tenant-configuration/parse_arguments.py similarity index 100% rename from multi-tenant-configuration/parsing_configurations.py rename to multi-tenant-configuration/parse_arguments.py From 009f96b9a4a35d41329cdd0f21afb947252ac7ef Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 23:02:00 +0200 Subject: [PATCH 72/79] Added config option to exclude certain tenants from checks --- multi-tenant-configuration/README.md | 3 ++- multi-tenant-configuration/config.py | 5 +++++ multi-tenant-configuration/main.py | 23 ++++++++++------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/multi-tenant-configuration/README.md b/multi-tenant-configuration/README.md index 221dc49..3b066b5 100644 --- a/multi-tenant-configuration/README.md +++ b/multi-tenant-configuration/README.md @@ -25,9 +25,10 @@ The script is configured by editing the values in `config.py`: | Configuration Key | Description | Default/Example | | :-------------------- | :-------------------------------------------- | :--------------------------- | -| `server_url` | The URL of the global admin node | `"http://localhost:8080"` | +| `server_url` | The URL of the global admin node | `"http://localhost:8080"` | | `tenant_url_pattern` | The URL pattern of the target tenants | `"http://{}:8080"` | | `tenant_urls` | Optional dictionary of server URLs per tenant | `{'tenant1': 'http://tenant1:8080', 'tenant2': 'http://tenant2:8080'}` | +| `ignored_tenants` | Optional list of tenants which are ignored | `['mh_default_org']` | | `digest_user` | The user name of the digest user | `opencast_system_account` | | `digest_pw` | The password of the digest user | `CHANGE_ME` | | `org_config_path` | The path to the organization config file | `"environment/{}/opencast-organizations.yml"` | diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index b3124b2..aadc478 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -15,6 +15,11 @@ # 'tenant2': 'http://tenant2:8080' # } +# List of tenants which should be ignored +ignored_tenants = [ + 'mh_default_org' +] + # Digest User login digest_user = "opencast_system_account" digest_pw = "CHANGE_ME" diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index b5d5828..dfbfc63 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -17,18 +17,19 @@ def main(): ### Parse args and config ### digest_login = DigestLogin(user=config.digest_user, password=config.digest_pw) - environment, tenant_id, check, verbose = parse_args() + environment, tenant_to_check, check, verbose = parse_args() logger = Logger(verbose) - # read and parse organization config - org_conf = read_yaml_file(config.org_config_path.format(environment)) + # parse script config config.tenant_ids = get_tenants(config.server_url, digest_login) - config.tenant_ids.remove('mh_default_org') + for ignored_tenant in config.ignored_tenants: + config.tenant_ids.remove(ignored_tenant) if not hasattr(config, 'tenant_urls'): config.tenant_urls = {} for tenant_id in config.tenant_ids: if not tenant_id in config.tenant_urls: config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) - # read group config + # read organization and group config + org_conf = read_yaml_file(config.org_config_path.format(environment)) group_config = read_yaml_file(config.group_config_path) # import config to scripts set_config_users(digest_login, org_conf, config, logger) @@ -36,19 +37,15 @@ def main(): set_config_capture_accounts(org_conf, config, logger) # if tenant is not given, we perform the checks for all tenants - tenants_to_check = [tenant_id] if tenant_id else config.tenant_ids + tenants_to_check = [tenant_to_check] if tenant_to_check else config.tenant_ids ### Start checks ### for tenant_id in tenants_to_check: - if check == 'all': - check_users(tenant_id) - check_groups(tenant_id) - check_capture_accounts(tenant_id) - elif check == 'users': + if check == 'users' or check == 'all': check_users(tenant_id) - elif check == 'groups': + if check == 'groups' or check == 'all': check_groups(tenant_id) - elif check == 'capture': + if check == 'capture' or check == 'all': check_capture_accounts(tenant_id) From 0753bdc1ab08421ce304b2f92beb2f566fe01cea Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 23:13:57 +0200 Subject: [PATCH 73/79] renamed variable --- multi-tenant-configuration/configure_capture_accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index 2fe18c3..d0f64b6 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -40,8 +40,8 @@ def check_capture_accounts(tenant_id: str): for organization in ENV_CONFIG['opencast_organizations']: # check switchcast system accounts if organization['id'] == tenant_id: - for capture_agent in organization['capture_agent_accounts']: - __check_capture_agent_account(capture_agent, tenant_id) + for capture_agent_account in organization['capture_agent_accounts']: + __check_capture_agent_account(capture_agent_account, tenant_id) def __check_capture_agent_account(account: dict, tenant_id: str): From ca6e2fd4989de846e0e9209048b800588553522c Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 23:15:13 +0200 Subject: [PATCH 74/79] changed Warning to Error --- multi-tenant-configuration/configure_capture_accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index d0f64b6..763bd09 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -58,9 +58,9 @@ def __check_capture_agent_account(account: dict, tenant_id: str): # check username and password if not account['username']: - print('WARNING: No Capture Agent Account has been configured') + print('ERROR: No Capture Agent Account has been configured') elif not account['password']: - print(f"WARNING: No password configured for Capture Agent User {account['username']}") + print(f"ERROR: No password configured for Capture Agent User {account['username']}") # Check if account has api access else: __check_access(account=account, tenant_id=tenant_id) From 1bd47ef662a706e541aaf1bf1c18bd139a6e1745 Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 23:36:49 +0200 Subject: [PATCH 75/79] merged two check functions into one --- .../configure_capture_accounts.py | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index 763bd09..11aa116 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -44,11 +44,20 @@ def check_capture_accounts(tenant_id: str): __check_capture_agent_account(capture_agent_account, tenant_id) -def __check_capture_agent_account(account: dict, tenant_id: str): +def __check_capture_agent_account(account: dict, tenant_id: str) -> bool: """ - Performs all checks for the specified Capture Agent Account: - - checks if account has API access (and if password matches) + Performs checks for the specified Capture Agent Account: - checks if username and password exists + - checks if account has API access (and if password matches) + Checks if the capture agent defined in the config has access to the service registry + with the username and password defined in the config, and sends a get request to '/services/available.json' + to find the ingest service. If check fails, prints a warning. + :param account: The user defined in the config + :type account: dict + :param tenant_id: The target tenant + :type tenant_id: String + :return: bool + :param account: The Capture Agent Account to be checked :type account: dict :param tenant_id: The target tenant @@ -59,33 +68,16 @@ def __check_capture_agent_account(account: dict, tenant_id: str): # check username and password if not account['username']: print('ERROR: No Capture Agent Account has been configured') - elif not account['password']: + return False + if not account['password']: print(f"ERROR: No password configured for Capture Agent User {account['username']}") - # Check if account has api access - else: - __check_access(account=account, tenant_id=tenant_id) - - -def __check_access(account: dict, tenant_id: str) -> bool: - """ - Checks if the capture agent defined in the config has access to the ingest service. - The check tries to access the ingest service with the username and password defined in the config, - and sends a get request to '/services/available.json' . - If check fails, prints a warning. - :param account: The user defined in the config - :type account: dict - :param tenant_id: The target tenant - :type tenant_id: String - :return: bool - """ - log(f"Checking access for Capture Agent Account {account['username']}") + return False + # Check if account has api access url = f'{CONFIG.tenant_urls[tenant_id]}/services/available.json?serviceType=org.opencastproject.ingest' login = BasicLogin(user=account['username'], password=account['password']) - try: - response = get_request(url, login, '/services/available.json?serviceType=org.opencastproject.ingest', - use_digest=False) + response = get_request(url, login, '/services/available.json', use_digest=False) except RequestError: print(f"WARNING: Capture Agent {account['username']} has no access.") return False From 140b2441bfb8a3d6b942cd85df2503ac646d068e Mon Sep 17 00:00:00 2001 From: mheyen Date: Mon, 30 Aug 2021 23:40:49 +0200 Subject: [PATCH 76/79] removed redundant docs --- multi-tenant-configuration/configure_capture_accounts.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index 11aa116..b0590d6 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -52,16 +52,12 @@ def __check_capture_agent_account(account: dict, tenant_id: str) -> bool: Checks if the capture agent defined in the config has access to the service registry with the username and password defined in the config, and sends a get request to '/services/available.json' to find the ingest service. If check fails, prints a warning. - :param account: The user defined in the config - :type account: dict - :param tenant_id: The target tenant - :type tenant_id: String - :return: bool :param account: The Capture Agent Account to be checked :type account: dict :param tenant_id: The target tenant :type tenant_id: str + :return: bool """ log(f"Checking Capture Agent Account {account['username']} on tenant {tenant_id}.") From 39b2a19c899e409f330ce22e046c6554b621d7ae Mon Sep 17 00:00:00 2001 From: mheyen Date: Tue, 31 Aug 2021 21:57:18 +0200 Subject: [PATCH 77/79] moved env folder into config folder --- multi-tenant-configuration/config.py | 2 +- .../environment/staging/opencast-organizations.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename multi-tenant-configuration/{ => configurations}/environment/staging/opencast-organizations.yml (100%) diff --git a/multi-tenant-configuration/config.py b/multi-tenant-configuration/config.py index aadc478..d794ff7 100644 --- a/multi-tenant-configuration/config.py +++ b/multi-tenant-configuration/config.py @@ -26,6 +26,6 @@ # path to environment configuration file. # The {} are a placeholder which will be filled with the environment passed as an argument (e.g. staging or production). -org_config_path = "environment/{}/opencast-organizations.yml" +org_config_path = "configurations/environment/{}/opencast-organizations.yml" # path to group configuration file group_config_path = "configurations/group_configuration.yaml" diff --git a/multi-tenant-configuration/environment/staging/opencast-organizations.yml b/multi-tenant-configuration/configurations/environment/staging/opencast-organizations.yml similarity index 100% rename from multi-tenant-configuration/environment/staging/opencast-organizations.yml rename to multi-tenant-configuration/configurations/environment/staging/opencast-organizations.yml From 4cab9543c894936237ca4ed2621915bf9eb17a0f Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 1 Sep 2021 10:52:33 +0200 Subject: [PATCH 78/79] Restructuring: Iterating over organizations is now done in main.py --- .../configure_capture_accounts.py | 15 +++----- multi-tenant-configuration/configure_users.py | 38 +++++++++++-------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/multi-tenant-configuration/configure_capture_accounts.py b/multi-tenant-configuration/configure_capture_accounts.py index b0590d6..dbb3f8f 100644 --- a/multi-tenant-configuration/configure_capture_accounts.py +++ b/multi-tenant-configuration/configure_capture_accounts.py @@ -28,20 +28,17 @@ def set_config_capture_accounts(env_conf: dict, config: dict, logger: Logger): log = logger.log -def check_capture_accounts(tenant_id: str): +def check_capture_accounts(tenant): """ Performs the checks for each capture agent on the specified tenant - :param tenant_id: The target tenant - :type tenant_id: str + :param tenant: The target tenant + :type tenant: dict """ - log('\nStart checking Capture Agent Accounts for tenant: ', tenant_id) + log('\nStart checking Capture Agent Accounts for tenant: ', tenant['id']) # Check and configure Capture Agent Accounts: - for organization in ENV_CONFIG['opencast_organizations']: - # check switchcast system accounts - if organization['id'] == tenant_id: - for capture_agent_account in organization['capture_agent_accounts']: - __check_capture_agent_account(capture_agent_account, tenant_id) + for capture_agent_account in tenant['capture_agent_accounts']: + __check_capture_agent_account(capture_agent_account, tenant['id']) def __check_capture_agent_account(account: dict, tenant_id: str) -> bool: diff --git a/multi-tenant-configuration/configure_users.py b/multi-tenant-configuration/configure_users.py index a8b8f9c..ff76917 100644 --- a/multi-tenant-configuration/configure_users.py +++ b/multi-tenant-configuration/configure_users.py @@ -38,26 +38,32 @@ def set_config_users(digest_login: DigestLogin, env_conf: dict, config: dict, lo log = logger.log -def check_users(tenant_id: str): +def check_system_accounts(system_accounts, tenant_id): """ - Performs the checks for each user on the specified tenant + Performs checks on the system accounts (e.g. player, annotate, cast). + + :param system_accounts: The switchcast system accounts to be checked + :type system_accounts: dict :param tenant_id: The target tenant :type tenant_id: str """ - log('\nStart checking users for tenant: ', tenant_id) - - # Check and configure System User Accounts & External API User Accounts: - for organization in ENV_CONFIG['opencast_organizations']: - # check switchcast system accounts - if organization['id'] == 'All Tenants': - log(f'Checking system accounts for tenant {tenant_id} ...') - for system_account in organization['switchcast_system_accounts']: - __check_user(system_account, tenant_id) - # check and configure external api accounts - if organization['id'] == tenant_id: - log(f'Checking External API accounts for tenant {tenant_id} ...') - for user in organization['external_api_accounts']: - __check_user(user, tenant_id) + # check switchcast system accounts + log(f"Start checking system accounts for tenant {tenant_id} ...") + for system_account in system_accounts: + __check_user(system_account, tenant_id) + + +def check_external_api_accounts(tenant): + """ + Performs checks on the external api accounts for the given tenant. + + :param tenant: The target tenant + :type tenant: dict + """ + # check and configure external api accounts + log(f"Start checking External API accounts for tenant {tenant['id']} ...") + for user in tenant['external_api_accounts']: + __check_user(user, tenant['id']) def __check_user(user: dict, tenant_id: str): From ddc70ca95e0f2515ac75dda426bfe4daae5b6e09 Mon Sep 17 00:00:00 2001 From: mheyen Date: Wed, 1 Sep 2021 10:55:56 +0200 Subject: [PATCH 79/79] Addition to former commit - Iteration is done in main.py --- multi-tenant-configuration/main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/multi-tenant-configuration/main.py b/multi-tenant-configuration/main.py index dfbfc63..744a06c 100644 --- a/multi-tenant-configuration/main.py +++ b/multi-tenant-configuration/main.py @@ -7,7 +7,7 @@ from input_output.yaml_utils import read_yaml_file from rest_requests.basic_requests import get_tenants from parse_arguments import parse_args -from configure_users import check_users, set_config_users +from configure_users import check_external_api_accounts, check_system_accounts, set_config_users from configure_groups import check_groups, set_config_groups from configure_capture_accounts import check_capture_accounts, set_config_capture_accounts import config @@ -28,8 +28,12 @@ def main(): for tenant_id in config.tenant_ids: if not tenant_id in config.tenant_urls: config.tenant_urls[tenant_id] = config.tenant_url_pattern.format(tenant_id) - # read organization and group config + # read and parse organization config org_conf = read_yaml_file(config.org_config_path.format(environment)) + opencast_organizations = {} + for organization in org_conf['opencast_organizations']: + opencast_organizations[organization['id']] = organization + # read group config group_config = read_yaml_file(config.group_config_path) # import config to scripts set_config_users(digest_login, org_conf, config, logger) @@ -42,11 +46,12 @@ def main(): ### Start checks ### for tenant_id in tenants_to_check: if check == 'users' or check == 'all': - check_users(tenant_id) + check_system_accounts(opencast_organizations['All Tenants']['switchcast_system_accounts'], tenant_id) + check_external_api_accounts(opencast_organizations[tenant_id]) if check == 'groups' or check == 'all': check_groups(tenant_id) if check == 'capture' or check == 'all': - check_capture_accounts(tenant_id) + check_capture_accounts(opencast_organizations[tenant_id]) if __name__ == '__main__':