From 71d5762aaff9d98579d7e1e1bd408b522b4a19a7 Mon Sep 17 00:00:00 2001 From: Igor Mateus Omote Date: Tue, 11 Jul 2023 19:05:09 -0300 Subject: [PATCH 1/4] feat(test): update yml with data test structure --- warehouse/dbt_project.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/warehouse/dbt_project.yml b/warehouse/dbt_project.yml index b4ca39a..8ca4fe4 100644 --- a/warehouse/dbt_project.yml +++ b/warehouse/dbt_project.yml @@ -31,5 +31,19 @@ models: staging: +schema: staging +tests: + data: + tests_high_prior: + severity: error + error_if: "!=0" + tests_medium_prior: + severity: error + error_if: ">10000" + warn_if: ">100" + tests_low_prior: + severity: warn + error_if: ">100" + warn_if: "!=0" + vars: sandbox_ids: ['ac06b512-0726-4c7c-b143-2a4a0a3c4467', 'dd63b8b4-8872-4fed-a33a-12a22a3fa6f2', '4866ebb1-39f2-4656-a556-a822ff85ea5c', '542a7c6b-b18d-4152-abae-f43ea445366f', '2db3f19e-3322-43d3-95f8-df12c7e8b9a7', 'a7f60032-72fc-41a8-bb41-ada0944fb512', 'fb0872cd-85ff-47dd-bdb5-2c8dddfdb916', 'bd4d7747-d70b-4f63-aa88-fd60d8008381'] From e15b06da8974a2b8f5c007177a0fcd4f94f2c214 Mon Sep 17 00:00:00 2001 From: Igor Mateus Omote Date: Tue, 11 Jul 2023 19:05:14 -0300 Subject: [PATCH 2/4] feat(test): create 3 initial tests --- .../llm_io_operations_duration.sql | 32 +++++++++++++ .../llm_open_ai_request_duration.sql | 35 +++++++++++++++ .../llm_window_size_threshold.sql | 45 +++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 warehouse/tests/tests_low_prior/llm_io_operations_duration.sql create mode 100644 warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql create mode 100644 warehouse/tests/tests_low_prior/llm_window_size_threshold.sql diff --git a/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql b/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql new file mode 100644 index 0000000..5765cca --- /dev/null +++ b/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql @@ -0,0 +1,32 @@ +with openai_stats as ( + select *, + extract(epoch from (requested_at - created_at)) as db_write_duration, + extract(epoch from (responded_at - requested_at)) as openai_duration + from {{ ref('stg_llm__open_ai_completion_status') }} +) +select + action, + entity, + model, + -- Database write duration percentiles + (percentile_cont(0.50) within group (order by db_write_duration))::decimal(8, 2) as db_write_p50, + (percentile_cont(0.95) within group (order by db_write_duration))::decimal(8, 2) as db_write_p95, + (percentile_cont(0.99) within group (order by db_write_duration))::decimal(8, 2) as db_write_p99, + -- OpenAI request duration percentiles + (percentile_cont(0.50) within group (order by openai_duration))::decimal(8, 2) as openai_p50, + (percentile_cont(0.95) within group (order by openai_duration))::decimal(8, 2) as openai_p95, + (percentile_cont(0.99) within group (order by openai_duration))::decimal(8, 2) as openai_p99, + -- OpenAI consumed tokens percentiles + (percentile_cont(0.50) within group (order by consumed_tokens))::int as consumed_tokens_p50, + (percentile_cont(0.95) within group (order by consumed_tokens))::int as consumed_tokens_p95, + (percentile_cont(0.99) within group (order by consumed_tokens))::int as consumed_tokens_p99, + -- OpenAI produced tokens percentiles + (percentile_cont(0.50) within group (order by produced_tokens))::int as produced_tokens_p50, + (percentile_cont(0.95) within group (order by produced_tokens))::int as produced_tokens_p95, + (percentile_cont(0.99) within group (order by produced_tokens))::int as produced_tokens_p99 +from + openai_stats +group by + action, + entity, + model diff --git a/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql new file mode 100644 index 0000000..db32c5e --- /dev/null +++ b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql @@ -0,0 +1,35 @@ +select + width_bucket( + extract ( + epoch + from + responded_at - requested_at + ), + 0, + 60, + 60 + ) as duration_in_seconds, + count(*) +from + {{ ref('stg_llm__open_ai_completion_status') }} +where + requested_at is not null + and responded_at is not null +group by + duration_in_seconds +order by + duration_in_seconds; + +-- OpenAI total tokens usage histogram for analytical purposes. At first, does not require any alarms +select + width_bucket(total_tokens, 0, 4000, 10) * 400 as total_tokens_bucket, + count(*) +from + {{ ref('stg_llm__open_ai_completion_status') }} +where + total_tokens is not null + and "model" = 'gpt-3.5-turbo' +group by + total_tokens_bucket +order by + total_tokens_bucket; diff --git a/warehouse/tests/tests_low_prior/llm_window_size_threshold.sql b/warehouse/tests/tests_low_prior/llm_window_size_threshold.sql new file mode 100644 index 0000000..414c44f --- /dev/null +++ b/warehouse/tests/tests_low_prior/llm_window_size_threshold.sql @@ -0,0 +1,45 @@ +with context_window as ( + select + * + from + ( + values + ('gpt-3.5-turbo', 4000), + ('gpt-3.5-turbo-16k', 16000), + ('gpt-4', 8000), + ('gpt-4-32k', 32000) + ) as context_window("model", "window_size") +), +threshold as ( + select + * + from + ( + values + ('normal', 0, 0.50), + ('look_out_50%', 0.50, 0.75), + ('warning_75%', 0.75, 0.90), + ('severe_90%', 0.90, 0.95), + ('critical_95%', 0.95, 1), + ('exceeded_100%', 1, float8 '+infinity') + ) as threshold("label", "min", "max") +) +select + oac.model as model, + t.label as threshold, + count(*) as "count" +from + {{ ref('stg_llm__open_ai_completion_status') }} oac + inner join context_window cw on cw.model = oac.model + inner join threshold t on oac.total_tokens :: float / cw.window_size :: float between t.min + and t.max - 0.00001 +where + oac.total_tokens is not null + and oac.model = cw.model + and t.label <> 'normal' +group by + oac.model, + t.label, + t.min +order by + t.min From 6af2849c3fca7682cfa05ccf97e251a8b3e47d7c Mon Sep 17 00:00:00 2001 From: Igor Mateus Omote Date: Tue, 11 Jul 2023 19:11:14 -0300 Subject: [PATCH 3/4] fix(tests): split test and add doc to them --- .../tests_low_prior/llm_io_operations_duration.sql | 1 + .../llm_open_ai_request_duration.sql | 14 +------------- .../tests_low_prior/llm_open_ai_token_usage.sql | 13 +++++++++++++ .../llm_window_size_threshold.sql | 1 + 4 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql rename warehouse/tests/{tests_low_prior => tests_medium_prior}/llm_window_size_threshold.sql (94%) diff --git a/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql b/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql index 5765cca..e5e0a51 100644 --- a/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql +++ b/warehouse/tests/tests_low_prior/llm_io_operations_duration.sql @@ -1,3 +1,4 @@ +-- IO operations durations for analytical purposes. At first, does not require any alarms with openai_stats as ( select *, extract(epoch from (requested_at - created_at)) as db_write_duration, diff --git a/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql index db32c5e..df64a0f 100644 --- a/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql +++ b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql @@ -1,3 +1,4 @@ +-- OpenAI request duration histogram for analytical purposes. At first, does not require any alarms select width_bucket( extract ( @@ -20,16 +21,3 @@ group by order by duration_in_seconds; --- OpenAI total tokens usage histogram for analytical purposes. At first, does not require any alarms -select - width_bucket(total_tokens, 0, 4000, 10) * 400 as total_tokens_bucket, - count(*) -from - {{ ref('stg_llm__open_ai_completion_status') }} -where - total_tokens is not null - and "model" = 'gpt-3.5-turbo' -group by - total_tokens_bucket -order by - total_tokens_bucket; diff --git a/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql b/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql new file mode 100644 index 0000000..cbadb40 --- /dev/null +++ b/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql @@ -0,0 +1,13 @@ +-- OpenAI total tokens usage histogram for analytical purposes. At first, does not require any alarms +select + width_bucket(total_tokens, 0, 4000, 10) * 400 as total_tokens_bucket, + count(*) +from + {{ ref('stg_llm__open_ai_completion_status') }} +where + total_tokens is not null + and "model" = 'gpt-3.5-turbo' +group by + total_tokens_bucket +order by + total_tokens_bucket; diff --git a/warehouse/tests/tests_low_prior/llm_window_size_threshold.sql b/warehouse/tests/tests_medium_prior/llm_window_size_threshold.sql similarity index 94% rename from warehouse/tests/tests_low_prior/llm_window_size_threshold.sql rename to warehouse/tests/tests_medium_prior/llm_window_size_threshold.sql index 414c44f..346d04c 100644 --- a/warehouse/tests/tests_low_prior/llm_window_size_threshold.sql +++ b/warehouse/tests/tests_medium_prior/llm_window_size_threshold.sql @@ -1,3 +1,4 @@ +-- Total tokens that exceed each model's context window token size limit with context_window as ( select * From ec85bee3380ebbb9512088ddb2e19c7d83b8866a Mon Sep 17 00:00:00 2001 From: Igor Mateus Omote Date: Tue, 11 Jul 2023 19:13:29 -0300 Subject: [PATCH 4/4] fix(tests): remove semicolon --- .../tests/tests_low_prior/llm_open_ai_request_duration.sql | 3 +-- warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql index df64a0f..649d1f0 100644 --- a/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql +++ b/warehouse/tests/tests_low_prior/llm_open_ai_request_duration.sql @@ -19,5 +19,4 @@ where group by duration_in_seconds order by - duration_in_seconds; - + duration_in_seconds diff --git a/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql b/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql index cbadb40..f87b46f 100644 --- a/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql +++ b/warehouse/tests/tests_low_prior/llm_open_ai_token_usage.sql @@ -10,4 +10,4 @@ where group by total_tokens_bucket order by - total_tokens_bucket; + total_tokens_bucket