From c0a12438ea4f572efe52a50659ea78976170e792 Mon Sep 17 00:00:00 2001 From: steve lasker Date: Fri, 4 Oct 2024 13:50:29 -0700 Subject: [PATCH 1/3] Update to getting started docs Signed-off-by: steve lasker --- .vscode/settings.json | 6 + conserver/configuring-links.md | 86 ++++++++ conserver/conserver-quick-start.md | 336 ++++++++++++----------------- conserver/inside-the-conserver.md | 3 +- conserver/standard-links.md | 5 +- 5 files changed, 236 insertions(+), 200 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 conserver/configuring-links.md diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3f419f2 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "cSpell.words": [ + "conserver", + "vcons" + ] +} \ No newline at end of file diff --git a/conserver/configuring-links.md b/conserver/configuring-links.md new file mode 100644 index 0000000..b9156ed --- /dev/null +++ b/conserver/configuring-links.md @@ -0,0 +1,86 @@ +--- +description: Configuring Conserver Links +--- + +# Configuring Conserver Links + +Links are extensibility points to the conserver. +A conserver is configured with one or more links. +A collection of links construct a chain, which is executed in sequence. + +Configuration is saved in the `vcon-server/config.yml` file. + +## links: + +A collection of links, loaded from the folder specified in `module:`, and configured under the `options:` node. + +## storage: + +Storage drivers for vCons + +## chains: + +The sequence of links to execute when processing a vCon + +### ingress_lists: + +The queue to monitor for processing by the link + +### egress_lists: + +The queue to move the vcon upon successful processing. +Chains can be sequenced by configuring the `egress_lists:` to reference the same name as another `chain`'s `ingress_lists`. + + ```yaml + links: + : + module: links. + options: + "name": "value-pairs" + "instructions": "in each link folder" + expire_vcon: + module: links.expire_vcon + options: + seconds: 604800 + summarize: + module: links.analyze + options: + OPENAI_API_KEY: xxxxx + prompt: "Summarize this transcript in a few sentences, identify the purpose and the parties of the conversation. Mention if there was a voicemail or if the customer and agent spoke." + analysis_type: summary + model: 'gpt-4o-mini' + webhook_store_call_log: + module: links.webhook + options: + webhook-urls: + - https://example.com/conserver + storages: + + chains: + my-chain: + links: + - expire_vcon + - summarize + ingress_lists: + - default_ingress + storages: + egress_lists: + - default_egress + enabled: 1 + summarize_only: + links: + - summarize + ingress_lists: + - summarize_ingress + egress_lists: + - summarize_egress + notification-chain: + links: + - webhook_store_call_log + ingress_lists: + - default_egress + storages: + egress_lists: + - end-queue + enabled: 1 + ``` diff --git a/conserver/conserver-quick-start.md b/conserver/conserver-quick-start.md index 5886372..b548d7c 100644 --- a/conserver/conserver-quick-start.md +++ b/conserver/conserver-quick-start.md @@ -4,218 +4,156 @@ description: A quick start to getting the conserver up and running # 🐰 Conserver Quick Start -## Ubuntu Install - -Based on a digital ocean install, to keep it vanilla. Created a 4 GB Memory / 2 Intel vCPUs / 120 GB Disk / NYC3 - Ubuntu 23.04 x64 droplet, logged in. - -``` -snap install docker -git clone https://github.com/vcon-dev/vcon.git -cd vcon/ -git submodule sync -git submodule update --init --recursive -``` - -Create an \~/vcon/.env file for some of the global environmental stuff. See example .env below. - -## Conserver Start - -The conserver repo can be downloaded directly, but is also included in the vcon repo as a sub-repo in the von-server directory. - -``` -cd vcon-server -``` - -Secrets for the conserver are kept in the .env file at the root of the vcon\_server directory. - -## Example vcon-server/.env file - -``` -AWS_BUCKET=vcon-storage -AWS_KEY_ID=xxxxx -AWS_SECRET_KEY=xxxxx -DEEPGRAM_KEY=xxxxx -ENV=dev - -# CORE DEPENDENCIES -ENV=dev -HOSTNAME=http://0.0.0.0:8000 -HOST=0.0.0.0 -PORT=8000 -REDIS_URL=redis://redis - -# Overriding these on pairing so they don't conflict with django port etc -REDIS_EXTERNAL_PORT=8001 -CONSERVER_EXTERNAL_PORT=8000 - -CONSERVER_API_TOKEN=1111111 -CONSERVER_CONFIG_FILE=./config.yml -``` - -Create a new config file in the server directory - -## Example vcon-server/config.yml - -``` -links: - webhook_store_call_log: - module: links.webhook - options: - webhook-urls: - - https://example.com/conserver - expire_vcon: - module: links.expire_vcon - options: - seconds: 604800 - expire_vcon_in_10_minutes: - module: links.expire_vcon - options: - seconds: 600 - deepgram: - module: links.deepgram - options: - DEEPGRAM_KEY: xxxxxxxxxxxx - minimum_duration: 30 - api: - model: "nova-2" - smart_format: true - detect_language: true - summarize: - module: links.analyze - options: - OPENAI_API_KEY: xxxxx - prompt: "Summarize this transcript in a few sentences, identify the purpose and the parties of the conversation. Mention if there was a voicemail or if the customer and agent spoke." - analysis_type: summary - model: 'gpt-4o-mini' - sentiment: - module: links.analyze - options: - OPENAI_API_KEY: xxxx - prompt: "Based on this transcript - if the customer complained, if the customer said they were angry or disappointed, if the customer threatened or used profanity, respond with only the words 'NEEDS REVIEW', otherwise respond 'NO REVIEW NEEDED'." - analysis_type: customer_frustration - model: 'gpt-4o-mini' - diarize: - module: links.analyze - options: - OPENAI_API_KEY: xxxx - prompt: "Go step by step: 1. Diarize the conversation and also identify the Agent and the Customer and show the names along with it like Agent(Agent Name) and output in markdown and label each speaker in bold. Don't add any extra information except for the speakers. Don't add -the word markdown. 2. If it's only one speaker, return the transcript. 3. If you can't diarize the transcript, return an empty string." - analysis_type: diarized - model: 'gpt-4o' - send_frustration_for_review: - module: links.post_analysis_to_slack - options: - token: xoxb-739777144080-xxxxxxxxxxx - default_channel_name: team-rainbow-alerts - url: "https:/www.moredetails.com/ca8ae4f5-0423-4b02-9975-42ed4e3eb155/latest" - analysis_to_post: summary - only_if: +The quickstart will cover: + +- [Installing the conserver with Docker](#ubuntu-install-wdocker) +- [Viewing the default configuration](#conserver-configuration) +- [Staring a conserver for processing](#start-the-conserver) +- [Monitoring realtime logs](#viewing-logs-realtime) +- [Submitting a vCon](#submit-a-vcon) +- [Process a vCon](#process-a-chain-of-vcons) + +## Ubuntu Install w/Docker + +- Install the [Docker client](https://www.docker.com/get-started/), or equivalent + + ```bash + snap install docker + ``` + +- Clone the vCon Repos + + ```bash + git clone https://github.com/vcon-dev/vcon.git + cd vcon/ + git submodule sync + git submodule update --init --recursive + ``` + +## Conserver Configuration + +A conserver has two configuration files: + +- Service Host & Secret Configuration: `vcon-server/.env` +- Service Processing Configuration: `vcon-server/config.yml` + +The conserver repo (`/vcon-server`) can be downloaded directly, but is also included in the `vcon` repo cloned above as a sub-repo in the vcon-server directory. + +- Configure secrets in the `vcon_server/.env` directory. + For the purposes of getting started in a local container, no changes are needed. + + **Example `vcon-server/.env` file** + + ```bash + # CORE DEPENDENCIES + ENV=dev + HOSTNAME=http://0.0.0.0:8000 + HOST=0.0.0.0 + PORT=8000 + REDIS_URL=redis://redis + + # Overriding these on pairing so they don't conflict with django port etc + REDIS_EXTERNAL_PORT=8001 + CONSERVER_EXTERNAL_PORT=8000 + + CONSERVER_API_TOKEN=1111111 + CONSERVER_CONFIG_FILE=./config.yml + ``` + +- View the Conserver config file in the server directory. + See [Configuring Links](./configuring-links.md) for more details. + + **Example `vcon-server/config.yml`** + + ```yaml + links: + deepgram: + module: links.deepgram + options: + DEEPGRAM_KEY: xxxxxxxxxxxx + minimum_duration: 30 + api: + model: "nova-2" + smart_format: true + detect_language: true + diarize: + module: links.analyze + options: + OPENAI_API_KEY: xxxx + prompt: "Go step by step: 1. Diarize the conversation and also identify the Agent and the Customer and show the names along with it like Agent(Agent Name) and output in markdown and label each speaker in bold. Don't add any extra information except for the speakers. Don't add the word markdown. 2. If it's only one speaker, return the transcript. 3. If you can't diarize the transcript, return an empty string." + analysis_type: diarized + model: 'gpt-4o' + sentiment: + module: links.analyze + options: + OPENAI_API_KEY: xxxx + prompt: "Based on this transcript - if the customer complained, if the customer said they were angry or disappointed, if the customer threatened or used profanity, respond with only the words 'NEEDS REVIEW', otherwise respond 'NO REVIEW NEEDED'." analysis_type: customer_frustration - includes: NEEDS REVIEW -storages: - postgres: - module: storage.postgres - options: - user: postgres - password: xxxxxxxx - host: xxxxxx.us-east-1.rds.amazonaws.com - port: "5432" - database: postgres - s3: - module: storage.s3 - options: - aws_access_key_id: xxxxx - aws_secret_access_key: xxxx - aws_bucket: vcons - elasticsearch: - module: storage.elasticsearch - options: - cloud_id: "xxxxx:xxxx==" - api_key: "xxxxxxx==" - index: vcon_index - -chains: - bria_chain: - links: - - deepgram + model: 'gpt-4o-mini' + summarize: + module: links.analyze + options: + OPENAI_API_KEY: xxxxx + prompt: "Summarize this transcript in a few sentences, identify the purpose and the parties of the conversation. Mention if there was a voicemail or if the customer and agent spoke." + analysis_type: summary + model: 'gpt-4o-mini' + storages: + + chains: + my-chain: + links: + - deepgram + - diarize + - sentiment + - summarize + ingress_lists: + - default_ingress + storages: + egress_lists: + - default_egress + enabled: 1 + + summarize_only: + links: - summarize - - sentiment - - diarize - - agent_note - - webhook_store_call_log - - send_frustration_for_review - - expire_vcon - ingress_lists: - - default_ingress - storages: - - postgres - - s3 - - elasticsearch - egress_lists: - - default_egress - enabled: 1 - - volie_chain: - links: - - expire_vcon - ingress_lists: - - volie_ingress - storages: - - postgres - - s3 - egress_lists: - - volie_egress - enabled: 1 - - elastic_only: - links: - - expire_vcon - ingress_lists: - - elastic_ingress - storages: - - elasticsearch - - # This is to fix some old vcons (incorrect lead attachments and etc) - store_and_expire: - links: - - expire_vcon_in_10_minutes - ingress_lists: - - store_and_expire_ingress - storages: - - postgres - - s3 - - elasticsearch - -``` + ingress_lists: + - summarize_ingress + egress_lists: + - summarize_egress + ``` ### Start the Conserver -``` +Using a local Docker configuration, start + +```bash docker network create conserver -docker compose build -docker compose up -docker compose up --scale conserver=4 -d +docker compose up -d ``` -## Troubleshooting and Checking +### Troubleshooting and Checking -You can validate that the conserver is running on the command line using "docker ps". In the example below, we can see four instances running. +You can validate that the conserver is running on the command line using `docker ps`. +In the example below, we can see: -``` -root@partner-demo:~/vcon/vcon-server# docker ps +- `vcon-server-conserver-1`: processes vcons +- `vcon-server-api-1`: ingests requests +- `vcon-server-redis-1`: default storage and persistance + +```output +# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -21bc6e3aacd7 vcon-server-conserver "/app/docker/wait_fo…" 4 minutes ago Up 4 minutes vcon-server-conserver-4 -2e3a0341043d vcon-server-conserver "/app/docker/wait_fo…" 4 minutes ago Up 4 minutes vcon-server-conserver-2 -9c699287f035 vcon-server-conserver "/app/docker/wait_fo…" 4 minutes ago Up 4 minutes vcon-server-conserver-3 ffe6f68941c8 vcon-server-conserver "/app/docker/wait_fo…" 5 minutes ago Up 5 minutes vcon-server-conserver-1 8136e15912c5 vcon-server-api "/app/docker/wait_fo…" 5 minutes ago Up 5 minutes 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp vcon-server-api-1 e3388b5f23be redis/redis-stack:latest "/entrypoint.sh" 5 minutes ago Up 5 minutes (healthy) 6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp vcon-server-redis-1 -root@partner-demo:~/vcon/vcon-server# ``` -You can see the operational logs using "docker compose logs -f". Here's a typical log: +## Viewing Logs Realtime -``` +Monitor operational logs using `docker compose logs -f`. +A typical log: + +```output vcon-server-redis-1 | 9:C 23 Aug 2024 17:27:20.581 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. vcon-server-redis-1 | 9:C 23 Aug 2024 17:27:20.582 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo vcon-server-redis-1 | 9:C 23 Aug 2024 17:27:20.582 * Redis version=7.4.0, bits=64, commit=00000000, modified=0, pid=9, just started @@ -283,8 +221,12 @@ vcon-server-api-1 | {"asctime": "2024-08-23 17:27:24,227", "levelname": " vcon-server-conserver-1 | Redis is ready! vcon-server-conserver-1 | Redis is ready. Starting the dependent service... vcon-server-conserver-1 | {"asctime": "2024-08-23 17:27:22,240", "levelname": "INFO", "name": "__main__", "message": "Starting main loop", "taskName": null} - ``` -The [vCon admin program](https://github.com/vcon-dev/vcon-admin) is a nice tool for managing the conserver. +## Submit a vCon + +## Process a Chain of vCons + +## Postman Configuration +The [vCon admin program](https://github.com/vcon-dev/vcon-admin) is a nice tool for managing the conserver. diff --git a/conserver/inside-the-conserver.md b/conserver/inside-the-conserver.md index e9c489e..f6106bf 100644 --- a/conserver/inside-the-conserver.md +++ b/conserver/inside-the-conserver.md @@ -69,7 +69,7 @@ The [main loop of the conserver](https://github.com/vcon-dev/vcon-server/blob/86 * When an item (vCon ID) is found, it creates a VconChainRequest and processes it. * Handles exceptions by moving problematic vCons to a Dead Letter Queue. -Step by step: +Step by Step: 1. Processing starts when vCon UUIDs are placed into a ingress list. Chains may have several ingress lists, and have to have at least one to kick off processing. Lists are implemented as REDIS lists, and processing is controlled at the thread layer by blocking until the a new element is placed on the list. UUIDs can be added to the ingress list by other chains, allowing them to be placed in series, from links that can request processing, or from the API. A typical pattern is to create the vCon using the API, then inserting the UUID into the desired ingress list. 2. For each vCon taken from the ingress list, it is processed by each link in the chain. This `_process_link` function handles the execution of a single link in the processing chain for a vCon. Here's a summary of its functionality: @@ -98,4 +98,3 @@ REDIS is responsible for storing the conversations, while FAST API coordinates t Each vcon is stored in REDIS using JSON and named with a regular key: vcon:\{{vcon-uuid\}}, as are chains "chains:\{{name\}}", links "link:\{{name\}}" and storages "storage:\{{name\}}}". REDIS allows for the addition of dedicated hardware to accelerate long running and high compute use cases such as transcription and video redaction, as these systems can connect directly to REDIS relieving scale issues from general purpose hardware, while managing the overhead of moving large amounts of data. Links take a vCon ID as inputs, and bear the responsibility of reading vCons if required, or giving them the option to hand off to optimized hardware. FAST API provides the application infrastructure for the conserver. The transformation steps are developed as Python modules and loaded as tasks managed by FAST API. As each task finishes, it notifies other system elements by publishing UUID of the vCon. Other tasks wait on these notifications, and when they receive the notification, they can act on that same vCon for whatever purpose they may have. In addition, FAST API provides a REST API to the store of vCons, and a simple UI to manage the conserver. - diff --git a/conserver/standard-links.md b/conserver/standard-links.md index 7874a86..209e013 100644 --- a/conserver/standard-links.md +++ b/conserver/standard-links.md @@ -1,6 +1,9 @@ # 🔗 Standard Links -The conserver ships with a standard set of links. Custom links can be installed by adding them into the "links" subdirectory. The link represents an operation on a vCon, such as redaction, sharing and analysis. The conserver can have many different configurations of the same kind of link, making them the essential unit of functionality. +The conserver ships with a standard set of links. +Custom links can be installed by adding them into the "links" subdirectory. +The link represents an operation on a vCon, such as redaction, sharing and analysis. +The conserver can have many different configurations of the same kind of link, making them the essential unit of functionality.
From dd48ef8ba85ac7d137be062a738ad659ea29f2af Mon Sep 17 00:00:00 2001 From: steve lasker Date: Fri, 4 Oct 2024 15:05:41 -0700 Subject: [PATCH 2/3] Add submitting and processing vcons Signed-off-by: steve lasker --- conserver/conserver-quick-start.md | 60 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/conserver/conserver-quick-start.md b/conserver/conserver-quick-start.md index b548d7c..60ea7ff 100644 --- a/conserver/conserver-quick-start.md +++ b/conserver/conserver-quick-start.md @@ -39,8 +39,8 @@ A conserver has two configuration files: The conserver repo (`/vcon-server`) can be downloaded directly, but is also included in the `vcon` repo cloned above as a sub-repo in the vcon-server directory. -- Configure secrets in the `vcon_server/.env` directory. - For the purposes of getting started in a local container, no changes are needed. +- Review the HOst & Secrets configuration in `vcon_server/.env`. + For the purposes of getting started in a local container, no changes are necessary. **Example `vcon-server/.env` file** @@ -60,7 +60,7 @@ The conserver repo (`/vcon-server`) can be downloaded directly, but is also incl CONSERVER_CONFIG_FILE=./config.yml ``` -- View the Conserver config file in the server directory. +- View the Conserver config file in the server directory. See [Configuring Links](./configuring-links.md) for more details. **Example `vcon-server/config.yml`** @@ -124,7 +124,13 @@ The conserver repo (`/vcon-server`) can be downloaded directly, but is also incl ### Start the Conserver -Using a local Docker configuration, start +- Run from the vcon-server directory + + ```bash + cd vcon-server/ + ``` + +- Start the services with a local Docker configuration ```bash docker network create conserver @@ -223,10 +229,56 @@ vcon-server-conserver-1 | Redis is ready. Starting the dependent service... vcon-server-conserver-1 | {"asctime": "2024-08-23 17:27:22,240", "levelname": "INFO", "name": "__main__", "message": "Starting main loop", "taskName": null} ``` +## Configure Postman + +A Postman workspaces provides examples for the subsequent steps. +The equivalent steps can be run with `curl` or other API clients. + +View the [Conserver Postman Collection](https://datatrails-inc.postman.co/workspace/Conserver~318b4e24-2e1f-464b-9da4-919135ef4dd3/request/3276815-c7a61c87-9ef7-4b29-b727-6c043bba6608?tab=body) + ## Submit a vCon +To process a vCon, it must first be submitted to the vCon server. + +A set of sample vCons are located at: https://github.com/vcon-dev/fake-vcons/ + +NOTE: Use vcons in the https://github.com/vcon-dev/fake-vcons/tree/main/2024/05/10 directory for compatibility with these quickstart docs. + +Paste the contents of a vCon into the body, formatted as json. + +Note recent versions of conserver require the first element: `"vcon": "0.0.1",` + +```bash +headers: "x-conserver-api-token": "1111111" +body: +{ + "vcon": "0.0.1", + "uuid": "bbba043b-d1aa-4691-8739-ac3ddd030382", + "created_at": "2024-05-07T16:33:29.004994", + ... +POST http://localhost:8000/vcon +``` + ## Process a Chain of vCons +To process a chain of links: + +- Post to: `http://localhost:8000/vcon/ingress` +- specify the ingress queue: `?ingress_list=default_ingress` in the query parameters. +- Specify one or more `vcon_uuid`'s in the body. + Note: the body is a json array. + +```bash +headers: "x-conserver-api-token": "1111111" +body: +[ + "bbba043b-d1aa-4691-8739-ac3ddd030382" +] +POST http://localhost:8000/vcon/ingress?ingress_list=default_ingress +``` + +Note the output in the console where the docker logs are monitored. + ## Postman Configuration The [vCon admin program](https://github.com/vcon-dev/vcon-admin) is a nice tool for managing the conserver. From 4f67c8e6d318ea539f71746dd36dbcfcb5bcae88 Mon Sep 17 00:00:00 2001 From: steve lasker Date: Thu, 17 Oct 2024 15:14:38 -0700 Subject: [PATCH 3/3] move logs to individual line Signed-off-by: steve lasker --- conserver/conserver-quick-start.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conserver/conserver-quick-start.md b/conserver/conserver-quick-start.md index 60ea7ff..26a2c82 100644 --- a/conserver/conserver-quick-start.md +++ b/conserver/conserver-quick-start.md @@ -156,7 +156,12 @@ e3388b5f23be redis/redis-stack:latest "/entrypoint.sh" 5 minutes ago ## Viewing Logs Realtime -Monitor operational logs using `docker compose logs -f`. +Monitor operational logs using: + +```bash +docker compose logs -f` +``` + A typical log: ```output