diff --git a/bin/k8s/Chart.yaml b/bin/k8s/Chart.yaml index ee57a2b8427..b212d229e9c 100644 --- a/bin/k8s/Chart.yaml +++ b/bin/k8s/Chart.yaml @@ -51,6 +51,12 @@ dependencies: version: 16.5.6 repository: https://charts.bitnami.com/bitnami + - name: postgresql + version: 16.5.6 + repository: https://charts.bitnami.com/bitnami + alias: postgresql-litellm + condition: litellm.persistence.enabled + - name: minio version: 15.0.7 repository: https://charts.bitnami.com/bitnami diff --git a/bin/k8s/templates/access-control-service-deployment.yaml b/bin/k8s/templates/access-control-service-deployment.yaml index a332c65bd39..fc6784c5189 100644 --- a/bin/k8s/templates/access-control-service-deployment.yaml +++ b/bin/k8s/templates/access-control-service-deployment.yaml @@ -46,6 +46,15 @@ spec: secretKeyRef: name: {{ .Release.Name }}-postgresql key: postgres-password +{{- if .Values.litellm.enabled }} + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-secret + key: LITELLM_MASTER_KEY + - name: LITELLM_BASE_URL + value: "http://{{ .Values.litellm.name }}-svc:{{ .Values.litellm.service.port }}" +{{- end }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/external-names.yaml b/bin/k8s/templates/external-names.yaml index 259c5fa6956..c20d2545fe1 100644 --- a/bin/k8s/templates/external-names.yaml +++ b/bin/k8s/templates/external-names.yaml @@ -59,14 +59,34 @@ to access services in the main namespace using the same service names. --- {{/* PostgreSQL ExternalName */}} -{{- include "external-name-service" (dict +{{- include "external-name-service" (dict "name" (printf "%s-postgresql" .Release.Name) "namespace" $workflowComputingUnitPoolNamespace "externalName" (printf "%s-postgresql.%s.svc.cluster.local" .Release.Name $namespace) ) | nindent 0 }} --- -{{/* Webserver ExternalName */}} +{{/* PostgreSQL LiteLLM ExternalName */}} +{{- if .Values.litellm.persistence.enabled }} +{{- include "external-name-service" (dict + "name" (printf "%s-postgresql-litellm" .Release.Name) + "namespace" $workflowComputingUnitPoolNamespace + "externalName" (printf "%s-postgresql-litellm.%s.svc.cluster.local" .Release.Name $namespace) +) | nindent 0 }} + +--- +{{- end }} +{{/* LiteLLM service ExternalName */}} +{{- if .Values.litellm.enabled }} +{{- include "external-name-service" (dict + "name" (printf "%s-svc" .Values.litellm.name) + "namespace" $workflowComputingUnitPoolNamespace + "externalName" (printf "%s-svc.%s.svc.cluster.local" .Values.litellm.name $namespace) +) | nindent 0 }} + +--- +{{- end }} +{{/* Webserver ExternalName */} {{- include "external-name-service" (dict "name" (printf "%s-svc" .Values.webserver.name) "namespace" $workflowComputingUnitPoolNamespace diff --git a/bin/k8s/templates/litellm-config.yaml b/bin/k8s/templates/litellm-config.yaml new file mode 100644 index 00000000000..50dd33072ac --- /dev/null +++ b/bin/k8s/templates/litellm-config.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if .Values.litellm.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-config + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + model_list: + - model_name: claude-haiku-4.5 + litellm_params: + model: claude-haiku-4-5-20251001 + api_key: "os.environ/ANTHROPIC_API_KEY" + + general_settings: + {{- if .Values.litellm.persistence.enabled }} + master_key: "os.environ/LITELLM_MASTER_KEY" + {{- end }} + # Disable spend tracking and key management for simpler setup + disable_spend_logs: {{ not .Values.litellm.persistence.enabled }} +{{- end }} diff --git a/bin/k8s/templates/litellm-deployment.yaml b/bin/k8s/templates/litellm-deployment.yaml new file mode 100644 index 00000000000..6fcd9afd794 --- /dev/null +++ b/bin/k8s/templates/litellm-deployment.yaml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if .Values.litellm.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.litellm.name }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.litellm.replicaCount }} + selector: + matchLabels: + app: {{ .Values.litellm.name }} + template: + metadata: + labels: + app: {{ .Values.litellm.name }} + spec: + containers: + - name: litellm + image: {{ .Values.litellm.image.repository }}:{{ .Values.litellm.image.tag }} + imagePullPolicy: {{ .Values.litellm.image.pullPolicy }} + ports: + - containerPort: {{ .Values.litellm.service.port }} + name: http + env: + {{- if .Values.litellm.persistence.enabled }} + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: litellm-secret + key: DATABASE_URL + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-secret + key: LITELLM_MASTER_KEY + {{- end }} + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secret + key: ANTHROPIC_API_KEY + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secret + key: OPENAI_API_KEY + command: + - litellm + - --config + - /etc/litellm/config.yaml + - --port + - "{{ .Values.litellm.service.port }}" + volumeMounts: + - name: config + mountPath: /etc/litellm + readOnly: true + resources: + {{- toYaml .Values.litellm.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /health/liveliness + port: {{ .Values.litellm.service.port }} + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health/readiness + port: {{ .Values.litellm.service.port }} + initialDelaySeconds: 10 + periodSeconds: 5 + volumes: + - name: config + configMap: + name: litellm-config +{{- end }} diff --git a/bin/k8s/templates/litellm-secret.yaml b/bin/k8s/templates/litellm-secret.yaml new file mode 100644 index 00000000000..1cb5d2a68b5 --- /dev/null +++ b/bin/k8s/templates/litellm-secret.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if .Values.litellm.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: litellm-secret + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ANTHROPIC_API_KEY: {{ .Values.litellm.apiKeys.anthropic | b64enc | quote }} + OPENAI_API_KEY: {{ .Values.litellm.apiKeys.openai | b64enc | quote }} + {{- if .Values.litellm.persistence.enabled }} + {{- $postgresqlLitellm := index .Values "postgresql-litellm" }} + DATABASE_URL: {{ printf "postgresql://%s:%s@%s-postgresql-litellm:5432/%s" $postgresqlLitellm.auth.username $postgresqlLitellm.auth.password .Release.Name $postgresqlLitellm.auth.database | b64enc | quote }} + LITELLM_MASTER_KEY: {{ .Values.litellm.masterKey | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/bin/k8s/templates/litellm-service.yaml b/bin/k8s/templates/litellm-service.yaml new file mode 100644 index 00000000000..982ffabbec9 --- /dev/null +++ b/bin/k8s/templates/litellm-service.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if .Values.litellm.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.litellm.name }}-svc + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.litellm.service.type }} + selector: + app: {{ .Values.litellm.name }} + ports: + - protocol: TCP + port: {{ .Values.litellm.service.port }} + targetPort: {{ .Values.litellm.service.port }} + name: http +{{- end }} diff --git a/bin/k8s/templates/postgresql-litellm-persistence.yaml b/bin/k8s/templates/postgresql-litellm-persistence.yaml new file mode 100644 index 00000000000..027ae758b9e --- /dev/null +++ b/bin/k8s/templates/postgresql-litellm-persistence.yaml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{/* Define storage path configuration, please change it to your own path and make sure the path exists with the right permission*/}} +{{/* This path only works for local-path storage class */}} +{{- $hostBasePath := .Values.persistence.postgresqlHostLocalPath }} + +{{- if .Values.litellm.persistence.enabled }} +{{- $name := "postgresql-litellm" }} +{{- $persistence := (index .Values "postgresql-litellm").primary.persistence }} +{{- $volumeName := printf "%s-data-pv" $name }} +{{- $claimName := printf "%s-data-pvc" $name }} +{{- $storageClass := $persistence.storageClass | default "local-path" }} +{{- $size := $persistence.size | default "5Gi" }} +{{- $hostPath := printf "%s/%s/%s" $hostBasePath $.Release.Name $name }} + +{{/* Only create PV for local-path storage class */}} +{{- if and (eq $storageClass "local-path") (ne $hostBasePath "") }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ $volumeName }} + {{- if not $.Values.persistence.removeAfterUninstall }} + annotations: + "helm.sh/resource-policy": keep + {{- end }} + labels: + type: local + app: {{ $.Release.Name }} + component: {{ $name }} +spec: + storageClassName: {{ $storageClass }} + capacity: + storage: {{ $size }} + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ $hostPath }} +--- +{{- end }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $claimName }} + namespace: {{ $.Release.Namespace }} + {{- if not $.Values.persistence.removeAfterUninstall }} + annotations: + "helm.sh/resource-policy": keep + {{- end }} + labels: + app: {{ $.Release.Name }} + component: {{ $name }} +spec: + storageClassName: {{ $storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ $size }} + {{- if and (eq $storageClass "local-path") (ne $hostBasePath "") }} + volumeName: {{ $volumeName }} + {{- end }} +{{- end }} diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index f13fcb4a8c4..61678f231d9 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -293,6 +293,66 @@ pythonLanguageServer: cpu: "100m" memory: "100Mi" +# LiteLLM Proxy configuration +litellm: + enabled: true + name: litellm + replicaCount: 1 + image: + repository: ghcr.io/berriai/litellm-database + tag: main-stable + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 4000 + resources: + limits: + cpu: "2" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + # Database persistence configuration + persistence: + enabled: true + # Master key for LiteLLM admin API (must start with "sk-") + masterKey: "sk-texera-litellm-masterkey" # Change this in production + apiKeys: + # Set your Anthropic API key here + # IMPORTANT: In production, use external secrets management (e.g., sealed-secrets, external-secrets) + anthropic: "" # Replace with your actual API key or use external secret + # Set your OpenAI API key here + openai: "" # Replace with your actual API key or use external secret + +# PostgreSQL database for LiteLLM persistence +postgresql-litellm: + image: + repository: texera/postgres17-pgroonga + tag: latest + debug: true + auth: + postgresPassword: litellm_root_password # Change this in production + username: litellm + password: litellm_password # Should match litellm.persistence.database.password + database: litellm + primary: + livenessProbe: + initialDelaySeconds: 30 + readinessProbe: + initialDelaySeconds: 30 + resources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + persistence: + enabled: true + size: 5Gi + storageClass: local-path + existingClaim: "postgresql-litellm-data-pvc" + # Metrics Server configuration metrics-server: enabled: true # set to false if metrics-server is already installed @@ -349,6 +409,12 @@ ingressPaths: pathType: ImplementationSpecific serviceName: envoy-svc servicePort: 10000 + - path: /api/models + serviceName: texera-access-control-service-svc + servicePort: 9096 + - path: /api/chat + serviceName: texera-access-control-service-svc + servicePort: 9096 - path: /api serviceName: webserver-svc servicePort: 8080