diff --git a/assets/Screenshot 2025-11-23 023944.png b/assets/Screenshot 2025-11-23 023944.png new file mode 100644 index 0000000..f7ef14d Binary files /dev/null and b/assets/Screenshot 2025-11-23 023944.png differ diff --git a/assets/Screenshot 2025-11-23 024001.png b/assets/Screenshot 2025-11-23 024001.png new file mode 100644 index 0000000..c9fede6 Binary files /dev/null and b/assets/Screenshot 2025-11-23 024001.png differ diff --git a/assets/Screenshot 2025-11-23 024015.png b/assets/Screenshot 2025-11-23 024015.png new file mode 100644 index 0000000..bfdbff2 Binary files /dev/null and b/assets/Screenshot 2025-11-23 024015.png differ diff --git a/assets/Screenshot 2025-11-23 033554.png b/assets/Screenshot 2025-11-23 033554.png new file mode 100644 index 0000000..a605e19 Binary files /dev/null and b/assets/Screenshot 2025-11-23 033554.png differ diff --git a/infrastructure/storage/chromadb/37fb3117-dc7d-45ff-9fb6-bb48a0497e05/length.bin b/infrastructure/storage/chromadb/37fb3117-dc7d-45ff-9fb6-bb48a0497e05/length.bin index 1548bb0..6c7bf2d 100644 Binary files a/infrastructure/storage/chromadb/37fb3117-dc7d-45ff-9fb6-bb48a0497e05/length.bin and b/infrastructure/storage/chromadb/37fb3117-dc7d-45ff-9fb6-bb48a0497e05/length.bin differ diff --git a/infrastructure/storage/chromadb/chroma.sqlite3 b/infrastructure/storage/chromadb/chroma.sqlite3 index 853cd34..df640b1 100644 Binary files a/infrastructure/storage/chromadb/chroma.sqlite3 and b/infrastructure/storage/chromadb/chroma.sqlite3 differ diff --git a/openwebui/pipelines/.dockerignore b/openwebui/pipelines/.dockerignore new file mode 100644 index 0000000..a56310c --- /dev/null +++ b/openwebui/pipelines/.dockerignore @@ -0,0 +1,11 @@ +.venv +.env +.git +.gitignore +.github +Dockerfile +examples +docs +*.md +dev.sh +dev-docker.sh diff --git a/openwebui/pipelines/.github/FUNDING.yml b/openwebui/pipelines/.github/FUNDING.yml new file mode 100644 index 0000000..ef274fa --- /dev/null +++ b/openwebui/pipelines/.github/FUNDING.yml @@ -0,0 +1 @@ +github: tjbck diff --git a/openwebui/pipelines/.github/workflows/build-docker-image.yaml b/openwebui/pipelines/.github/workflows/build-docker-image.yaml new file mode 100644 index 0000000..7f2bd6e --- /dev/null +++ b/openwebui/pipelines/.github/workflows/build-docker-image.yaml @@ -0,0 +1,120 @@ +name: Build Docker Image + +on: + workflow_call: + inputs: + build_args: + required: false + default: "" + type: string + cache_id: + required: true + type: string + extract_flavor: + required: false + default: "" + type: string + image_name: + required: true + type: string + image_tag: + required: false + default: "" + type: string + registry: + required: false + default: ghcr.io + type: string + +env: + FULL_IMAGE_NAME: ${{ inputs.registry }}/${{ inputs.image_name }} + +jobs: + build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + ${{ inputs.image_tag }} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + ${{ inputs.extract_flavor }} + + - name: Extract metadata for Docker cache + id: cache-meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + flavor: | + prefix=cache-${{ inputs.cache_id }}-${{ matrix.platform }}- + + - name: Build Docker image + uses: docker/build-push-action@v5 + id: build + with: + context: . + push: true + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + build-args: | + BUILD_HASH=${{ github.sha }} + ${{ inputs.build_args }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ inputs.cache_id }}-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 diff --git a/openwebui/pipelines/.github/workflows/docker-build.yaml b/openwebui/pipelines/.github/workflows/docker-build.yaml new file mode 100644 index 0000000..62398b4 --- /dev/null +++ b/openwebui/pipelines/.github/workflows/docker-build.yaml @@ -0,0 +1,60 @@ +name: Create and publish Docker images with specific build args + +on: + workflow_dispatch: + push: + branches: + - main + - dev + +jobs: + build-main-image: + uses: ./.github/workflows/build-docker-image.yaml + with: + image_name: ${{ github.repository }} + cache_id: main + + build-cuda-image: + uses: ./.github/workflows/build-docker-image.yaml + with: + image_name: ${{ github.repository }} + cache_id: cuda + image_tag: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + extract_flavor: suffix=-cuda,onlatest=true + build_args: | + USE_CUDA=true + + build-minimum-image: + uses: ./.github/workflows/build-docker-image.yaml + with: + image_name: ${{ github.repository }} + cache_id: minimum + image_tag: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=minimum + extract_flavor: suffix=-minimum,onlatest=true + build_args: | + MINIMUM_BUILD=true + + merge-main-images: + uses: ./.github/workflows/merge-docker-images.yaml + needs: [build-main-image] + with: + image_name: ${{ github.repository }} + cache_id: main + + merge-cuda-images: + uses: ./.github/workflows/merge-docker-images.yaml + needs: [build-cuda-image] + with: + image_name: ${{ github.repository }} + cache_id: cuda + extract_flavor: suffix=-cuda,onlatest=true + extract_tags: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + + merge-minimum-images: + uses: ./.github/workflows/merge-docker-images.yaml + needs: [build-minimum-image] + with: + image_name: ${{ github.repository }} + cache_id: minimum + extract_flavor: suffix=-minimum,onlatest=true + extract_tags: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=minimum diff --git a/openwebui/pipelines/.github/workflows/merge-docker-images.yaml b/openwebui/pipelines/.github/workflows/merge-docker-images.yaml new file mode 100644 index 0000000..512f42c --- /dev/null +++ b/openwebui/pipelines/.github/workflows/merge-docker-images.yaml @@ -0,0 +1,71 @@ +name: Merge Docker Images + +on: + workflow_call: + inputs: + cache_id: + required: true + type: string + extract_flavor: + required: false + default: "" + type: string + extract_tags: + required: false + default: "" + type: string + image_name: + required: true + type: string + registry: + required: false + default: ghcr.io + type: string + +env: + FULL_IMAGE_NAME: ${{ inputs.registry }}/${{ inputs.image_name }} + +jobs: + merge-images: + runs-on: ubuntu-latest + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-${{ inputs.cache_id }}-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + ${{ inputs.extract_tags }} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + ${{ inputs.extract_flavor }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/openwebui/pipelines/.gitignore b/openwebui/pipelines/.gitignore new file mode 100644 index 0000000..d454a74 --- /dev/null +++ b/openwebui/pipelines/.gitignore @@ -0,0 +1,12 @@ +__pycache__ +.env + +/litellm + + +pipelines/* +!pipelines/.gitignore +.DS_Store + +.venv +venv/ \ No newline at end of file diff --git a/openwebui/pipelines/.webui_secret_key b/openwebui/pipelines/.webui_secret_key new file mode 100644 index 0000000..1a0f016 --- /dev/null +++ b/openwebui/pipelines/.webui_secret_key @@ -0,0 +1 @@ +o1jkf6Q9i+/33ijt \ No newline at end of file diff --git a/openwebui/pipelines/CONTRIBUTING.md b/openwebui/pipelines/CONTRIBUTING.md new file mode 100644 index 0000000..3fd6f03 --- /dev/null +++ b/openwebui/pipelines/CONTRIBUTING.md @@ -0,0 +1,50 @@ +## Contributing to Pipelines + +πŸš€ **Welcome, Contributors!** πŸš€ + +We are thrilled to have you join the Pipelines community! Your contributions are essential to making Pipelines a powerful and versatile framework for extending OpenAI-compatible applications' capabilities. This document provides guidelines to ensure your contributions are smooth and effective. + +### πŸ“Œ Key Points + +- **Scope of Pipelines:** Remember that Pipelines is a framework designed to enhance OpenAI interactions, specifically through a plugin-like approach. Focus your contributions on making Pipelines more robust, flexible, and user-friendly within this context. +- **Open WebUI Integration:** Pipelines is primarily designed to work with Open WebUI. While contributions that expand compatibility with other platforms are welcome, prioritize functionalities that seamlessly integrate with Open WebUI's ecosystem. + +### 🚨 Reporting Issues + +Encountered a bug or have an idea for improvement? We encourage you to report it! Here's how: + +1. **Check Existing Issues:** Browse the [Issues tab](https://github.com/open-webui/pipelines/issues) to see if the issue or suggestion has already been reported. +2. **Open a New Issue:** If it's a new issue, feel free to open one. Follow the issue template for clear and concise reporting. Provide detailed descriptions, steps to reproduce, expected outcomes, and actual results. This helps us understand and resolve the issue efficiently. + +### 🧭 Scope of Support + +- **Python Fundamentals:** Pipelines leverages Python. Basic Python knowledge is essential for contributing effectively. + +## πŸ’‘ Contributing + +Ready to make a difference? Here's how you can contribute to Pipelines: + +### πŸ›  Pull Requests + +We encourage pull requests to improve Pipelines! Here's the process: + +1. **Discuss Your Idea:** If your contribution involves significant changes, discuss it in the [Issues tab](https://github.com/open-webui/pipelines/issues) first. This ensures your idea aligns with the project's vision. +2. **Coding Standards:** Follow the project's coding standards and write clear, descriptive commit messages. +3. **Update Documentation:** If your contribution impacts documentation, update it accordingly. +4. **Submit Your Pull Request:** Submit your pull request and provide a clear summary of your changes. + +### πŸ“š Documentation + +Help make Pipelines more accessible by: + +- **Writing Tutorials:** Create guides for setting up, using, and customizing Pipelines. +- **Improving Documentation:** Enhance existing documentation for clarity, completeness, and accuracy. +- **Adding Examples:** Contribute pipelines examples that showcase different functionalities and use cases. + +### πŸ€” Questions & Feedback + +Got questions or feedback? Join our [Discord community](https://discord.gg/5rJgQTnV4s) or open an issue. We're here to help! + +## πŸ™ Thank You! + +Your contributions are invaluable to Pipelines' success! We are excited to see what you bring to the project. Together, we can create a powerful and versatile framework for extending OpenAI capabilities. 🌟 \ No newline at end of file diff --git a/openwebui/pipelines/Dockerfile b/openwebui/pipelines/Dockerfile new file mode 100644 index 0000000..9ee4aee --- /dev/null +++ b/openwebui/pipelines/Dockerfile @@ -0,0 +1,77 @@ +FROM python:3.11-slim-bookworm AS base + +# Use args +ARG MINIMUM_BUILD +ARG USE_CUDA +ARG USE_CUDA_VER +ARG PIPELINES_URLS +ARG PIPELINES_REQUIREMENTS_PATH + +## Basis ## +ENV ENV=prod \ + PORT=9099 \ + # pass build args to the build + MINIMUM_BUILD=${MINIMUM_BUILD} \ + USE_CUDA_DOCKER=${USE_CUDA} \ + USE_CUDA_DOCKER_VER=${USE_CUDA_VER} + +# Install GCC and build tools. +# These are kept in the final image to enable installing packages on the fly. +RUN apt-get update && \ + apt-get install -y gcc build-essential curl git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install Python dependencies +COPY ./requirements.txt . +COPY ./requirements-minimum.txt . +RUN pip3 install uv +RUN if [ "$MINIMUM_BUILD" != "true" ]; then \ + if [ "$USE_CUDA_DOCKER" = "true" ]; then \ + pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir; \ + else \ + pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir; \ + fi \ + fi +RUN if [ "$MINIMUM_BUILD" = "true" ]; then \ + uv pip install --system -r requirements-minimum.txt --no-cache-dir; \ + else \ + uv pip install --system -r requirements.txt --no-cache-dir; \ + fi + + +# Layer on for other components +FROM base AS app + +ENV PIPELINES_URLS=${PIPELINES_URLS} \ + PIPELINES_REQUIREMENTS_PATH=${PIPELINES_REQUIREMENTS_PATH} + +# Copy the application code +COPY . . + +# Fix write permissions for OpenShift / non-root users +RUN set -eux; \ + for d in /app /root /.local /.cache; do \ + mkdir -p "$d"; \ + done; \ + chgrp -R 0 /app /root /.local /.cache || true; \ + chmod -R g+rwX /app /root /.local /.cache || true; \ + find /app -type d -exec chmod g+s {} + || true; \ + find /root -type d -exec chmod g+s {} + || true; \ + find /.local -type d -exec chmod g+s {} + || true; \ + find /.cache -type d -exec chmod g+s {} + || true + +# Run a docker command if either PIPELINES_URLS or PIPELINES_REQUIREMENTS_PATH is not empty +RUN if [ -n "$PIPELINES_URLS" ] || [ -n "$PIPELINES_REQUIREMENTS_PATH" ]; then \ + echo "Running docker command with PIPELINES_URLS or PIPELINES_REQUIREMENTS_PATH"; \ + ./start.sh --mode setup; \ + fi + +# Expose the port +ENV HOST="0.0.0.0" +ENV PORT="9099" + +# if we already installed the requirements on build, we can skip this step on run +ENTRYPOINT [ "bash", "start.sh" ] diff --git a/openwebui/pipelines/LICENSE b/openwebui/pipelines/LICENSE new file mode 100644 index 0000000..e05cc0e --- /dev/null +++ b/openwebui/pipelines/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Timothy Jaeryang Baek + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/openwebui/pipelines/README.md b/openwebui/pipelines/README.md new file mode 100644 index 0000000..6d99b69 --- /dev/null +++ b/openwebui/pipelines/README.md @@ -0,0 +1,194 @@ +

+ Pipelines Logo +

+ +# Pipelines: UI-Agnostic OpenAI API Plugin Framework + +> [!TIP] +> **DO NOT USE PIPELINES!** +> +> If your goal is simply to add support for additional providers like Anthropic or basic filters, you likely don't need Pipelines . For those cases, Open WebUI Functions are a better fitβ€”it's built-in, much more convenient, and easier to configure. Pipelines, however, comes into play when you're dealing with computationally heavy tasks (e.g., running large models or complex logic) that you want to offload from your main Open WebUI instance for better performance and scalability. + + +Welcome to **Pipelines**, an [Open WebUI](https://github.com/open-webui) initiative. Pipelines bring modular, customizable workflows to any UI client supporting OpenAI API specs – and much more! Easily extend functionalities, integrate unique logic, and create dynamic workflows with just a few lines of code. + +## πŸš€ Why Choose Pipelines? + +- **Limitless Possibilities:** Easily add custom logic and integrate Python libraries, from AI agents to home automation APIs. +- **Seamless Integration:** Compatible with any UI/client supporting OpenAI API specs. (Only pipe-type pipelines are supported; filter types require clients with Pipelines support.) +- **Custom Hooks:** Build and integrate custom pipelines. + +### Examples of What You Can Achieve: + +- [**Function Calling Pipeline**](/examples/filters/function_calling_filter_pipeline.py): Easily handle function calls and enhance your applications with custom logic. +- [**Custom RAG Pipeline**](/examples/pipelines/rag/llamaindex_pipeline.py): Implement sophisticated Retrieval-Augmented Generation pipelines tailored to your needs. +- [**Message Monitoring Using Langfuse**](/examples/filters/langfuse_filter_pipeline.py): Monitor and analyze message interactions in real-time using Langfuse. +- [**Message Monitoring Using Opik**](/examples/filters/opik_filter_pipeline.py): Monitor and analyze message interactions using Opik, an open-source platform for debugging and evaluating LLM applications and RAG systems. +- [**Rate Limit Filter**](/examples/filters/rate_limit_filter_pipeline.py): Control the flow of requests to prevent exceeding rate limits. +- [**Real-Time Translation Filter with LibreTranslate**](/examples/filters/libretranslate_filter_pipeline.py): Seamlessly integrate real-time translations into your LLM interactions. +- [**Toxic Message Filter**](/examples/filters/detoxify_filter_pipeline.py): Implement filters to detect and handle toxic messages effectively. +- **And Much More!**: The sky is the limit for what you can accomplish with Pipelines and Python. [Check out our scaffolds](/examples/scaffolds) to get a head start on your projects and see how you can streamline your development process! + +## πŸ”§ How It Works + +

+ Pipelines Workflow +

+ +Integrating Pipelines with any OpenAI API-compatible UI client is simple. Launch your Pipelines instance and set the OpenAI URL on your client to the Pipelines URL. That's it! You're ready to leverage any Python library for your needs. + +## ⚑ Quick Start with Docker + +> [!WARNING] +> Pipelines are a plugin system with arbitrary code execution β€” **don't fetch random pipelines from sources you don't trust**. + +### Docker + +For a streamlined setup using Docker: + +1. **Run the Pipelines container:** + + ```sh + docker run -d -p 9099:9099 --add-host=host.docker.internal:host-gateway -v pipelines:/app/pipelines --name pipelines --restart always ghcr.io/open-webui/pipelines:main + ``` + +2. **Connect to Open WebUI:** + + - Navigate to the **Settings > Connections > OpenAI API** section in Open WebUI. + - Set the API URL to `http://localhost:9099` and the API key to `0p3n-w3bu!`. Your pipelines should now be active. + +> [!NOTE] +> If your Open WebUI is running in a Docker container, replace `localhost` with `host.docker.internal` in the API URL. + +3. **Manage Configurations:** + + - In the admin panel, go to **Admin Settings > Pipelines tab**. + - Select your desired pipeline and modify the valve values directly from the WebUI. + +> [!TIP] +> If you are unable to connect, it is most likely a Docker networking issue. We encourage you to troubleshoot on your own and share your methods and solutions in the discussions forum. + +If you need to install a custom pipeline with additional dependencies: + +- **Run the following command:** + + ```sh + docker run -d -p 9099:9099 --add-host=host.docker.internal:host-gateway -e PIPELINES_URLS="https://github.com/open-webui/pipelines/blob/main/examples/filters/detoxify_filter_pipeline.py" -v pipelines:/app/pipelines --name pipelines --restart always ghcr.io/open-webui/pipelines:main + ``` + +Alternatively, you can directly install pipelines from the admin settings by copying and pasting the pipeline URL, provided it doesn't have additional dependencies. + +That's it! You're now ready to build customizable AI integrations effortlessly with Pipelines. Enjoy! + +### Docker Compose together with Open WebUI + +Using [Docker Compose](https://docs.docker.com/compose/) simplifies the management of multi-container Docker applications. + +Here is an example configuration file `docker-compose.yaml` for setting up Open WebUI together with Pipelines using Docker Compose: + +```yaml +services: + openwebui: + image: ghcr.io/open-webui/open-webui:main + ports: + - "3000:8080" + volumes: + - open-webui:/app/backend/data + + pipelines: + image: ghcr.io/open-webui/pipelines:main + volumes: + - pipelines:/app/pipelines + restart: always + environment: + - PIPELINES_API_KEY=0p3n-w3bu! + +volumes: + open-webui: {} + pipelines: {} +``` + +To start your services, run the following command: + +``` +docker compose up -d +``` + +You can then use `http://pipelines:9099` (the name is the same as the service's name defined in `docker-compose.yaml`) as an API URL to connect to Open WebUI. + +> [!NOTE] +> The `pipelines` service is accessible only by `openwebui` Docker service and thus provide additional layer of security. + +## πŸ“¦ Installation and Setup + +Get started with Pipelines in a few easy steps: + +1. **Ensure Python 3.11 is installed.** +2. **Clone the Pipelines repository:** + + ```sh + git clone https://github.com/open-webui/pipelines.git + cd pipelines + ``` + +3. **Install the required dependencies:** + + ```sh + pip install -r requirements.txt + ``` + +4. **Start the Pipelines server:** + + ```sh + sh ./start.sh + ``` + +Once the server is running, set the OpenAI URL on your client to the Pipelines URL. This unlocks the full capabilities of Pipelines, integrating any Python library and creating custom workflows tailored to your needs. + +### Advanced Docker Builds +If you create your own pipelines, you can install them when the Docker image is built. For example, +create a bash script with the snippet below to collect files from a path, add them as install URLs, +and build the Docker image with the new pipelines automatically installed. + +NOTE: The pipelines module will still attempt to install any package dependencies found at in your +file headers at start time, but they will not be downloaded again. + +```sh +# build in the specific pipelines +PIPELINE_DIR="pipelines-custom" +# assuming the above directory is in your source repo and not skipped by `.dockerignore`, it will get copied to the image +PIPELINE_PREFIX="file:///app" + +# retrieve all the sub files +export PIPELINES_URLS= +for file in "$PIPELINE_DIR"/*; do + if [[ -f "$file" ]]; then + if [[ "$file" == *.py ]]; then + if [ -z "$PIPELINES_URLS" ]; then + PIPELINES_URLS="$PIPELINE_PREFIX/$file" + else + PIPELINES_URLS="$PIPELINES_URLS;$PIPELINE_PREFIX/$file" + fi + fi + fi +done +echo "New Custom Install Pipes: $PIPELINES_URLS" + +docker build --build-arg PIPELINES_URLS=$PIPELINES_URLS --build-arg MINIMUM_BUILD=true -f Dockerfile . +``` + +## πŸ“‚ Directory Structure and Examples + +The `/pipelines` directory is the core of your setup. Add new modules, customize existing ones, and manage your workflows here. All the pipelines in the `/pipelines` directory will be **automatically loaded** when the server launches. + +You can change this directory from `/pipelines` to another location using the `PIPELINES_DIR` env variable. + +### Integration Examples + +Find various integration examples in the `/examples` directory. These examples show how to integrate different functionalities, providing a foundation for building your own custom pipelines. + +## πŸŽ‰ Work in Progress + +We’re continuously evolving! We'd love to hear your feedback and understand which hooks and features would best suit your use case. Feel free to reach out and become a part of our Open WebUI community! + +Our vision is to push **Pipelines** to become the ultimate plugin framework for our AI interface, **Open WebUI**. Imagine **Open WebUI** as the WordPress of AI interfaces, with **Pipelines** being its diverse range of plugins. Join us on this exciting journey! 🌍 diff --git a/openwebui/pipelines/blueprints/function_calling_blueprint.py b/openwebui/pipelines/blueprints/function_calling_blueprint.py new file mode 100644 index 0000000..ad6a386 --- /dev/null +++ b/openwebui/pipelines/blueprints/function_calling_blueprint.py @@ -0,0 +1,189 @@ +from typing import List, Optional +from pydantic import BaseModel +from schemas import OpenAIChatMessage +import os +import requests +import json + +from utils.pipelines.main import ( + get_last_user_message, + add_or_update_system_message, + get_tools_specs, +) + +# System prompt for function calling +DEFAULT_SYSTEM_PROMPT = ( + """Tools: {} + +If a function tool doesn't match the query, return an empty string. Else, pick a +function tool, fill in the parameters from the function tool's schema, and +return it in the format {{ "name": \"functionName\", "parameters": {{ "key": +"value" }} }}. Only pick a function if the user asks. Only return the object. Do not return any other text." +""" + ) + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Valves for function calling + OPENAI_API_BASE_URL: str + OPENAI_API_KEY: str + TASK_MODEL: str + TEMPLATE: str + + def __init__(self, prompt: str | None = None) -> None: + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "function_calling_blueprint" + self.name = "Function Calling Blueprint" + self.prompt = prompt or DEFAULT_SYSTEM_PROMPT + self.tools: object = None + + # Initialize valves + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + "OPENAI_API_BASE_URL": os.getenv( + "OPENAI_API_BASE_URL", "https://api.openai.com/v1" + ), + "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "YOUR_OPENAI_API_KEY"), + "TASK_MODEL": os.getenv("TASK_MODEL", "gpt-3.5-turbo"), + "TEMPLATE": """Use the following context as your learned knowledge, inside XML tags. + + {{CONTEXT}} + + +When answer to user: +- If you don't know, just say that you don't know. +- If you don't know when you are not sure, ask for clarification. +Avoid mentioning that you obtained the information from the context. +And answer according to the language of the user's question.""", + } + ) + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # If title generation is requested, skip the function calling filter + if body.get("title", False): + return body + + print(f"pipe:{__name__}") + print(user) + + # Get the last user message + user_message = get_last_user_message(body["messages"]) + + # Get the tools specs + tools_specs = get_tools_specs(self.tools) + + prompt = self.prompt.format(json.dumps(tools_specs, indent=2)) + content = "History:\n" + "\n".join( + [ + f"{message['role']}: {message['content']}" + for message in body["messages"][::-1][:4] + ] + ) + f"Query: {user_message}" + + result = self.run_completion(prompt, content) + messages = self.call_function(result, body["messages"]) + + return {**body, "messages": messages} + + # Call the function + def call_function(self, result, messages: list[dict]) -> list[dict]: + if "name" not in result: + return messages + + function = getattr(self.tools, result["name"]) + function_result = None + try: + function_result = function(**result["parameters"]) + except Exception as e: + print(e) + + # Add the function result to the system prompt + if function_result: + system_prompt = self.valves.TEMPLATE.replace( + "{{CONTEXT}}", function_result + ) + + messages = add_or_update_system_message( + system_prompt, messages + ) + + # Return the updated messages + return messages + + return messages + + def run_completion(self, system_prompt: str, content: str) -> dict: + r = None + try: + # Call the OpenAI API to get the function response + r = requests.post( + url=f"{self.valves.OPENAI_API_BASE_URL}/chat/completions", + json={ + "model": self.valves.TASK_MODEL, + "messages": [ + { + "role": "system", + "content": system_prompt, + }, + { + "role": "user", + "content": content, + }, + ], + # TODO: dynamically add response_format? + # "response_format": {"type": "json_object"}, + }, + headers={ + "Authorization": f"Bearer {self.valves.OPENAI_API_KEY}", + "Content-Type": "application/json", + }, + stream=False, + ) + r.raise_for_status() + + response = r.json() + content = response["choices"][0]["message"]["content"] + + # Parse the function response + if content != "": + result = json.loads(content) + print(result) + return result + + except Exception as e: + print(f"Error: {e}") + + if r: + try: + print(r.json()) + except: + pass + + return {} diff --git a/openwebui/pipelines/config.py b/openwebui/pipelines/config.py new file mode 100644 index 0000000..28b1031 --- /dev/null +++ b/openwebui/pipelines/config.py @@ -0,0 +1,24 @@ +import os +import logging +#################################### +# Load .env file +#################################### + +try: + from dotenv import load_dotenv, find_dotenv + + load_dotenv(find_dotenv("./.env")) +except ImportError: + print("dotenv not installed, skipping...") + +# Define log levels dictionary +LOG_LEVELS = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL +} + +API_KEY = os.getenv("PIPELINES_API_KEY", "0p3n-w3bu!") +PIPELINES_DIR = os.getenv("PIPELINES_DIR", "./pipelines") diff --git a/openwebui/pipelines/dev-docker.sh b/openwebui/pipelines/dev-docker.sh new file mode 100644 index 0000000..c9d256a --- /dev/null +++ b/openwebui/pipelines/dev-docker.sh @@ -0,0 +1,9 @@ +# Removes any existing Open WebUI and Pipelines containers/ volumes - uncomment if you need a fresh start +# docker rm --force pipelines +# docker rm --force open-webui +# docker volume rm pipelines +# docker volume rm open-webui + +# Runs the containers with Ollama image for Open WebUI and the Pipelines endpoint in place +docker run -d -p 9099:9099 --add-host=host.docker.internal:host-gateway -v pipelines:/app/pipelines --name pipelines --restart always --env-file .env ghcr.io/open-webui/pipelines:latest +docker run -d -p 3000:8080 -p 11434:11434 --add-host=host.docker.internal:host-gateway -v ~/.ollama:/root/.ollama -v open-webui:/app/backend/data --name open-webui --restart always -e OPENAI_API_BASE_URL=http://host.docker.internal:9099 -e OPENAI_API_KEY=0p3n-w3bu! -e OLLAMA_HOST=0.0.0.0 ghcr.io/open-webui/open-webui:ollama \ No newline at end of file diff --git a/openwebui/pipelines/dev.sh b/openwebui/pipelines/dev.sh new file mode 100644 index 0000000..715aeca --- /dev/null +++ b/openwebui/pipelines/dev.sh @@ -0,0 +1,2 @@ +PORT="${PORT:-9099}" +uvicorn main:app --port $PORT --host 0.0.0.0 --forwarded-allow-ips '*' --reload \ No newline at end of file diff --git a/openwebui/pipelines/docker-compose.yaml b/openwebui/pipelines/docker-compose.yaml new file mode 100644 index 0000000..be9925b --- /dev/null +++ b/openwebui/pipelines/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + openwebui: + image: ghcr.io/open-webui/open-webui:main + ports: + - "3000:8080" + volumes: + - open-webui:/app/backend/data + + pipelines: + image: ghcr.io/open-webui/pipelines:main + volumes: + - pipelines:/app/pipelines + restart: always + environment: + - PIPELINES_API_KEY=0p3n-w3bu! + +volumes: + open-webui: {} + pipelines: {} \ No newline at end of file diff --git a/openwebui/pipelines/docs/CODE_OF_CONDUCT.md b/openwebui/pipelines/docs/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..1f27d32 --- /dev/null +++ b/openwebui/pipelines/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,75 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contribute to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their explicit permission +- **Spamming of any kind** +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all community spaces and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, spamming, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at hello@openwebui.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Temporary Ban + +**Community Impact**: Any violation of community standards, including but not limited to inappropriate language, unprofessional behavior, harassment, or spamming. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 2. Permanent Ban + +**Community Impact**: Repeated or severe violations of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/openwebui/pipelines/docs/SECURITY.md b/openwebui/pipelines/docs/SECURITY.md new file mode 100644 index 0000000..c6ececf --- /dev/null +++ b/openwebui/pipelines/docs/SECURITY.md @@ -0,0 +1,32 @@ +# Security Policy + +Our primary goal is to ensure the protection and confidentiality of sensitive data stored by users on Pipelines. Additionally, we aim to maintain a secure and trusted environment for executing Pipelines, which effectively function as a plugin system with arbitrary code execution capabilities. + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| main | :white_check_mark: | +| others | :x: | + +## Secure Pipelines Execution + +To mitigate risks associated with the Pipelines plugin system, we recommend the following best practices: + +1. **Trusted Sources**: Only fetch and execute Pipelines from trusted sources. Do not retrieve or run Pipelines from untrusted or unknown origins. + +2. **Fixed Versions**: Instead of pulling the latest version of a Pipeline, consider using a fixed, audited version to ensure stability and security. + +3. **Sandboxing**: Pipelines are executed in a sandboxed environment to limit their access to system resources and prevent potential harm. + +4. **Code Review**: All Pipelines undergo a thorough code review process before being approved for execution on our platform. + +5. **Monitoring**: We continuously monitor the execution of Pipelines for any suspicious or malicious activities. + +## Reporting a Vulnerability + +If you discover a security issue within our system, please notify us immediately via a pull request or contact us on discord. We take all security reports seriously and will respond promptly. + +## Product Security + +We regularly audit our internal processes and system's architecture for vulnerabilities using a combination of automated and manual testing techniques. We are committed to implementing Static Application Security Testing (SAST) and Software Composition Analysis (SCA) scans in our project to further enhance our security posture. \ No newline at end of file diff --git a/openwebui/pipelines/docs/images/header.png b/openwebui/pipelines/docs/images/header.png new file mode 100644 index 0000000..a63b0be Binary files /dev/null and b/openwebui/pipelines/docs/images/header.png differ diff --git a/openwebui/pipelines/docs/images/workflow.png b/openwebui/pipelines/docs/images/workflow.png new file mode 100644 index 0000000..8b6e9c8 Binary files /dev/null and b/openwebui/pipelines/docs/images/workflow.png differ diff --git a/openwebui/pipelines/examples/filters/conversation_turn_limit_filter.py b/openwebui/pipelines/examples/filters/conversation_turn_limit_filter.py new file mode 100644 index 0000000..bb31939 --- /dev/null +++ b/openwebui/pipelines/examples/filters/conversation_turn_limit_filter.py @@ -0,0 +1,64 @@ +import os +from typing import List, Optional +from pydantic import BaseModel +from schemas import OpenAIChatMessage +import time + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Valves for conversation turn limiting + target_user_roles: List[str] = ["user"] + max_turns: Optional[int] = None + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "conversation_turn_limit_filter_pipeline" + self.name = "Conversation Turn Limit Filter" + + self.valves = self.Valves( + **{ + "pipelines": os.getenv("CONVERSATION_TURN_PIPELINES", "*").split(","), + "max_turns": 10, + } + ) + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"pipe:{__name__}") + print(body) + print(user) + + if user.get("role", "admin") in self.valves.target_user_roles: + messages = body.get("messages", []) + if len(messages) > self.valves.max_turns: + raise Exception( + f"Conversation turn limit exceeded. Max turns: {self.valves.max_turns}" + ) + + return body diff --git a/openwebui/pipelines/examples/filters/datadog_filter_pipeline.py b/openwebui/pipelines/examples/filters/datadog_filter_pipeline.py new file mode 100644 index 0000000..af1d2de --- /dev/null +++ b/openwebui/pipelines/examples/filters/datadog_filter_pipeline.py @@ -0,0 +1,121 @@ +""" +title: DataDog Filter Pipeline +author: 0xThresh +date: 2024-06-06 +version: 1.0 +license: MIT +description: A filter pipeline that sends traces to DataDog. +requirements: ddtrace +environment_variables: DD_LLMOBS_AGENTLESS_ENABLED, DD_LLMOBS_ENABLED, DD_LLMOBS_APP_NAME, DD_API_KEY, DD_SITE +""" + +from typing import List, Optional +import os + +from utils.pipelines.main import get_last_user_message, get_last_assistant_message +from pydantic import BaseModel +from ddtrace.llmobs import LLMObs + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + # e.g. ["llama3:latest", "gpt-3.5-turbo"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Valves + dd_api_key: str + dd_site: str + ml_app: str + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "datadog_filter_pipeline" + self.name = "DataDog Filter" + + # Initialize + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + "dd_api_key": os.getenv("DD_API_KEY"), + "dd_site": os.getenv("DD_SITE", "datadoghq.com"), + "ml_app": os.getenv("ML_APP", "pipelines-test"), + } + ) + + # DataDog LLMOBS docs: https://docs.datadoghq.com/tracing/llm_observability/sdk/ + self.LLMObs = LLMObs() + self.llm_span = None + self.chat_generations = {} + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + self.set_dd() + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + self.LLMObs.flush() + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + self.set_dd() + pass + + def set_dd(self): + self.LLMObs.enable( + ml_app=self.valves.ml_app, + api_key=self.valves.dd_api_key, + site=self.valves.dd_site, + agentless_enabled=True, + integrations_enabled=True, + ) + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"inlet:{__name__}") + + self.llm_span = self.LLMObs.llm( + model_name=body["model"], + name=f"filter:{__name__}", + model_provider="open-webui", + session_id=body["chat_id"], + ml_app=self.valves.ml_app + ) + + self.LLMObs.annotate( + span = self.llm_span, + input_data = get_last_user_message(body["messages"]), + ) + + return body + + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"outlet:{__name__}") + + self.LLMObs.annotate( + span = self.llm_span, + output_data = get_last_assistant_message(body["messages"]), + ) + + self.llm_span.finish() + self.LLMObs.flush() + + return body diff --git a/openwebui/pipelines/examples/filters/detoxify_filter_pipeline.py b/openwebui/pipelines/examples/filters/detoxify_filter_pipeline.py new file mode 100644 index 0000000..73fc3a9 --- /dev/null +++ b/openwebui/pipelines/examples/filters/detoxify_filter_pipeline.py @@ -0,0 +1,83 @@ +""" +title: Detoxify Filter Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for filtering out toxic messages using the Detoxify library. +requirements: detoxify +""" + +from typing import List, Optional +from schemas import OpenAIChatMessage +from pydantic import BaseModel +from detoxify import Detoxify +import os + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + # e.g. ["llama3:latest", "gpt-3.5-turbo"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "detoxify_filter_pipeline" + self.name = "Detoxify Filter" + + # Initialize + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + } + ) + + self.model = None + + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + + self.model = Detoxify("original") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This filter is applied to the form data before it is sent to the OpenAI API. + print(f"inlet:{__name__}") + + print(body) + user_message = body["messages"][-1]["content"] + + # Filter out toxic messages + toxicity = self.model.predict(user_message) + print(toxicity) + + if toxicity["toxicity"] > 0.5: + raise Exception("Toxic message detected") + + return body diff --git a/openwebui/pipelines/examples/filters/dynamic_ollama_vision_filter_pipeline.py b/openwebui/pipelines/examples/filters/dynamic_ollama_vision_filter_pipeline.py new file mode 100644 index 0000000..9eb01d2 --- /dev/null +++ b/openwebui/pipelines/examples/filters/dynamic_ollama_vision_filter_pipeline.py @@ -0,0 +1,91 @@ +""" +title: Ollama Dynamic Vision Pipeline +author: Andrew Tait Gehrhardt +date: 2024-06-18 +version: 1.0 +license: MIT +description: A pipeline for dynamically processing images when current model is a text only model +requirements: pydantic, aiohttp +""" + +from typing import List, Optional +from pydantic import BaseModel +import json +import aiohttp +from utils.pipelines.main import get_last_user_message + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + vision_model: str = "llava" + ollama_base_url: str = "" + model_to_override: str = "" + + def __init__(self): + self.type = "filter" + self.name = "Interception Filter" + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + } + ) + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def process_images_with_llava(self, images: List[str], content: str, vision_model: str, ollama_base_url: str) -> str: + url = f"{ollama_base_url}/api/chat" + payload = { + "model": vision_model, + "messages": [ + { + "role": "user", + "content": content, + "images": images + } + ] + } + + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload) as response: + if response.status == 200: + content = [] + async for line in response.content: + data = json.loads(line) + content.append(data.get("message", {}).get("content", "")) + return "".join(content) + else: + print(f"Failed to process images with LLava, status code: {response.status}") + return "" + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"pipe:{__name__}") + + images = [] + + # Ensure the body is a dictionary + if isinstance(body, str): + body = json.loads(body) + + model = body.get("model", "") + + # Get the content of the most recent message + user_message = get_last_user_message(body["messages"]) + + if model in self.valves.model_to_override: + messages = body.get("messages", []) + for message in messages: + if "images" in message: + images.extend(message["images"]) + raw_llava_response = await self.process_images_with_llava(images, user_message, self.valves.vision_model,self.valves.ollama_base_url) + llava_response = f"REPEAT THIS BACK: {raw_llava_response}" + message["content"] = llava_response + message.pop("images", None) # This will safely remove the 'images' key if it exists + + return body diff --git a/openwebui/pipelines/examples/filters/function_calling_filter_pipeline.py b/openwebui/pipelines/examples/filters/function_calling_filter_pipeline.py new file mode 100644 index 0000000..5ea9957 --- /dev/null +++ b/openwebui/pipelines/examples/filters/function_calling_filter_pipeline.py @@ -0,0 +1,100 @@ +import os +import requests +from typing import Literal, List, Optional +from datetime import datetime + + +from blueprints.function_calling_blueprint import Pipeline as FunctionCallingBlueprint + + +class Pipeline(FunctionCallingBlueprint): + class Valves(FunctionCallingBlueprint.Valves): + # Add your custom parameters here + OPENWEATHERMAP_API_KEY: str = "" + pass + + class Tools: + def __init__(self, pipeline) -> None: + self.pipeline = pipeline + + def get_current_time( + self, + ) -> str: + """ + Get the current time. + + :return: The current time. + """ + + now = datetime.now() + current_time = now.strftime("%H:%M:%S") + return f"Current Time = {current_time}" + + def get_current_weather( + self, + location: str, + unit: Literal["metric", "fahrenheit"] = "fahrenheit", + ) -> str: + """ + Get the current weather for a location. If the location is not found, return an empty string. + + :param location: The location to get the weather for. + :param unit: The unit to get the weather in. Default is fahrenheit. + :return: The current weather for the location. + """ + + # https://openweathermap.org/api + + if self.pipeline.valves.OPENWEATHERMAP_API_KEY == "": + return "OpenWeatherMap API Key not set, ask the user to set it up." + else: + units = "imperial" if unit == "fahrenheit" else "metric" + params = { + "q": location, + "appid": self.pipeline.valves.OPENWEATHERMAP_API_KEY, + "units": units, + } + + response = requests.get( + "http://api.openweathermap.org/data/2.5/weather", params=params + ) + response.raise_for_status() # Raises an HTTPError for bad responses + data = response.json() + + weather_description = data["weather"][0]["description"] + temperature = data["main"]["temp"] + + return f"{location}: {weather_description.capitalize()}, {temperature}Β°{unit.capitalize()[0]}" + + def calculator(self, equation: str) -> str: + """ + Calculate the result of an equation. + + :param equation: The equation to calculate. + """ + + # Avoid using eval in production code + # https://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html + try: + result = eval(equation) + return f"{equation} = {result}" + except Exception as e: + print(e) + return "Invalid equation" + + def __init__(self): + super().__init__() + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "my_tools_pipeline" + self.name = "My Tools Pipeline" + self.valves = self.Valves( + **{ + **self.valves.model_dump(), + "pipelines": ["*"], # Connect to all pipelines + "OPENWEATHERMAP_API_KEY": os.getenv("OPENWEATHERMAP_API_KEY", ""), + }, + ) + self.tools = self.Tools(self) diff --git a/openwebui/pipelines/examples/filters/google_translation_filter_pipeline.py b/openwebui/pipelines/examples/filters/google_translation_filter_pipeline.py new file mode 100644 index 0000000..a8d31e3 --- /dev/null +++ b/openwebui/pipelines/examples/filters/google_translation_filter_pipeline.py @@ -0,0 +1,185 @@ +""" +title: Google Translate Filter +author: SimonOriginal +date: 2024-06-28 +version: 1.0 +license: MIT +description: This pipeline integrates Google Translate for automatic translation of user and assistant messages +without requiring an API key. It supports multilingual communication by translating based on specified source +and target languages. +""" + +import re +from typing import List, Optional +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import os +import time +import asyncio +from functools import lru_cache + +from utils.pipelines.main import get_last_user_message, get_last_assistant_message + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + source_user: Optional[str] = "auto" + target_user: Optional[str] = "en" + source_assistant: Optional[str] = "en" + target_assistant: Optional[str] = "uk" + + def __init__(self): + self.type = "filter" + self.name = "Google Translate Filter" + self.valves = self.Valves( + **{ + "pipelines": ["*"], + } + ) + + # Initialize translation cache + self.translation_cache = {} + self.code_blocks = [] # List to store code blocks + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + pass + + # @lru_cache(maxsize=128) # LRU cache to store translation results + def translate(self, text: str, source: str, target: str) -> str: + url = "https://translate.googleapis.com/translate_a/single" + params = { + "client": "gtx", + "sl": source, + "tl": target, + "dt": "t", + "q": text, + } + + try: + r = requests.get(url, params=params) + r.raise_for_status() + result = r.json() + translated_text = ''.join([sentence[0] for sentence in result[0]]) + return translated_text + except requests.exceptions.RequestException as e: + print(f"Network error: {e}") + time.sleep(1) + return self.translate(text, source, target) + except Exception as e: + print(f"Error translating text: {e}") + return text + + def split_text_around_table(self, text: str) -> List[str]: + table_regex = r'((?:^.*?\|.*?\n)+)(?=\n[^\|\s].*?\|)' + matches = re.split(table_regex, text, flags=re.MULTILINE) + + if len(matches) > 1: + return [matches[0], matches[1]] + else: + return [text, ""] + + def clean_table_delimiters(self, text: str) -> str: + # Remove extra spaces from table delimiters + return re.sub(r'(\|\s*-+\s*)+', lambda m: m.group(0).replace(' ', '-'), text) + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"inlet:{__name__}") + + messages = body["messages"] + user_message = get_last_user_message(messages) + + print(f"User message: {user_message}") + + # Find and store code blocks + code_block_regex = r'```[\s\S]+?```' + self.code_blocks = re.findall(code_block_regex, user_message) + # Replace code blocks with placeholders + user_message_no_code = re.sub(code_block_regex, '__CODE_BLOCK__', user_message) + + parts = self.split_text_around_table(user_message_no_code) + text_before_table, table_text = parts + + # Check translation cache for text before table + translated_before_table = self.translation_cache.get(text_before_table) + if translated_before_table is None: + translated_before_table = self.translate( + text_before_table, + self.valves.source_user, + self.valves.target_user, + ) + self.translation_cache[text_before_table] = translated_before_table + + translated_user_message = translated_before_table + table_text + + # Clean table delimiters + translated_user_message = self.clean_table_delimiters(translated_user_message) + + # Restore code blocks + for code_block in self.code_blocks: + translated_user_message = translated_user_message.replace('__CODE_BLOCK__', code_block, 1) + + print(f"Translated user message: {translated_user_message}") + + for message in reversed(messages): + if message["role"] == "user": + message["content"] = translated_user_message + break + + body = {**body, "messages": messages} + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"outlet:{__name__}") + + messages = body["messages"] + assistant_message = get_last_assistant_message(messages) + + print(f"Assistant message: {assistant_message}") + + # Find and store code blocks + code_block_regex = r'```[\s\S]+?```' + self.code_blocks = re.findall(code_block_regex, assistant_message) + # Replace code blocks with placeholders + assistant_message_no_code = re.sub(code_block_regex, '__CODE_BLOCK__', assistant_message) + + parts = self.split_text_around_table(assistant_message_no_code) + text_before_table, table_text = parts + + # Check translation cache for text before table + translated_before_table = self.translation_cache.get(text_before_table) + if translated_before_table is None: + translated_before_table = self.translate( + text_before_table, + self.valves.source_assistant, + self.valves.target_assistant, + ) + self.translation_cache[text_before_table] = translated_before_table + + translated_assistant_message = translated_before_table + table_text + + # Clean table delimiters + translated_assistant_message = self.clean_table_delimiters(translated_assistant_message) + + # Restore code blocks + for code_block in self.code_blocks: + translated_assistant_message = translated_assistant_message.replace('__CODE_BLOCK__', code_block, 1) + + print(f"Translated assistant message: {translated_assistant_message}") + + for message in reversed(messages): + if message["role"] == "assistant": + message["content"] = translated_assistant_message + break + + body = {**body, "messages": messages} + return body diff --git a/openwebui/pipelines/examples/filters/home_assistant_filter.py b/openwebui/pipelines/examples/filters/home_assistant_filter.py new file mode 100644 index 0000000..a86ed9f --- /dev/null +++ b/openwebui/pipelines/examples/filters/home_assistant_filter.py @@ -0,0 +1,116 @@ +""" +title: HomeAssistant Filter Pipeline +author: Andrew Tait Gehrhardt +date: 2024-06-15 +version: 1.0 +license: MIT +description: A pipeline for controlling Home Assistant entities based on their easy names. Only supports lights at the moment. +requirements: pytz, difflib +""" +import requests +from typing import Literal, Dict, Any +from datetime import datetime +import pytz +from difflib import get_close_matches + +from blueprints.function_calling_blueprint import Pipeline as FunctionCallingBlueprint + +class Pipeline(FunctionCallingBlueprint): + class Valves(FunctionCallingBlueprint.Valves): + HOME_ASSISTANT_URL: str = "" + HOME_ASSISTANT_TOKEN: str = "" + + class Tools: + def __init__(self, pipeline) -> None: + self.pipeline = pipeline + + def get_current_time(self) -> str: + """ + Get the current time in EST. + + :return: The current time in EST. + """ + now_est = datetime.now(pytz.timezone('US/Eastern')) # Get the current time in EST + current_time = now_est.strftime("%I:%M %p") # %I for 12-hour clock, %M for minutes, %p for am/pm + return f"ONLY RESPOND 'Current time is {current_time}'" + + def get_all_lights(self) -> Dict[str, Any]: + """ + Lists my lights. + Shows me my lights. + Get a dictionary of all lights in my home. + + :return: A dictionary of light entity names and their IDs. + """ + if not self.pipeline.valves.HOME_ASSISTANT_URL or not self.pipeline.valves.HOME_ASSISTANT_TOKEN: + return {"error": "Home Assistant URL or token not set, ask the user to set it up."} + else: + url = f"{self.pipeline.valves.HOME_ASSISTANT_URL}/api/states" + headers = { + "Authorization": f"Bearer {self.pipeline.valves.HOME_ASSISTANT_TOKEN}", + "Content-Type": "application/json", + } + + response = requests.get(url, headers=headers) + response.raise_for_status() # Raises an HTTPError for bad responses + data = response.json() + + lights = {entity["attributes"]["friendly_name"]: entity["entity_id"] + for entity in data if entity["entity_id"].startswith("light.")} + + return lights + + def control_light(self, name: str, state: Literal['on', 'off']) -> str: + """ + Turn a light on or off based on its name. + + :param name: The friendly name of the light. + :param state: The desired state ('on' or 'off'). + :return: The result of the operation. + """ + if not self.pipeline.valves.HOME_ASSISTANT_URL or not self.pipeline.valves.HOME_ASSISTANT_TOKEN: + return "Home Assistant URL or token not set, ask the user to set it up." + + # Normalize the light name by converting to lowercase and stripping extra spaces + normalized_name = " ".join(name.lower().split()) + + # Get a dictionary of all lights + lights = self.get_all_lights() + if "error" in lights: + return lights["error"] + + # Find the closest matching light name + light_names = list(lights.keys()) + closest_matches = get_close_matches(normalized_name, light_names, n=1, cutoff=0.6) + + if not closest_matches: + return f"Light named '{name}' not found." + + best_match = closest_matches[0] + light_id = lights[best_match] + + url = f"{self.pipeline.valves.HOME_ASSISTANT_URL}/api/services/light/turn_{state}" + headers = { + "Authorization": f"Bearer {self.pipeline.valves.HOME_ASSISTANT_TOKEN}", + "Content-Type": "application/json", + } + payload = { + "entity_id": light_id + } + + response = requests.post(url, headers=headers, json=payload) + if response.status_code == 200: + return f"ONLY RESPOND 'Will do' TO THE USER. DO NOT SAY ANYTHING ELSE!" + else: + return f"ONLY RESPOND 'Couldn't find light' TO THE USER. DO NOT SAY ANYTHING ELSE!" + + def __init__(self): + super().__init__() + self.name = "My Tools Pipeline" + self.valves = self.Valves( + **{ + **self.valves.model_dump(), + "pipelines": ["*"], # Connect to all pipelines + }, + ) + self.tools = self.Tools(self) diff --git a/openwebui/pipelines/examples/filters/langfuse_filter_pipeline.py b/openwebui/pipelines/examples/filters/langfuse_filter_pipeline.py new file mode 100644 index 0000000..2d73e5d --- /dev/null +++ b/openwebui/pipelines/examples/filters/langfuse_filter_pipeline.py @@ -0,0 +1,333 @@ +""" +title: Langfuse Filter Pipeline +author: open-webui +date: 2025-06-16 +version: 1.7.1 +license: MIT +description: A filter pipeline that uses Langfuse. +requirements: langfuse<3.0.0 +""" + +from typing import List, Optional +import os +import uuid +import json + +from utils.pipelines.main import get_last_assistant_message +from pydantic import BaseModel +from langfuse import Langfuse +from langfuse.api.resources.commons.errors.unauthorized_error import UnauthorizedError + + +def get_last_assistant_message_obj(messages: List[dict]) -> dict: + """Retrieve the last assistant message from the message list.""" + for message in reversed(messages): + if message["role"] == "assistant": + return message + return {} + + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + secret_key: str + public_key: str + host: str + # New valve that controls whether task names are added as tags: + insert_tags: bool = True + # New valve that controls whether to use model name instead of model ID for generation + use_model_name_instead_of_id_for_generation: bool = False + debug: bool = False + + def __init__(self): + self.type = "filter" + self.name = "Langfuse Filter" + + self.valves = self.Valves( + **{ + "pipelines": ["*"], + "secret_key": os.getenv("LANGFUSE_SECRET_KEY", "your-secret-key-here"), + "public_key": os.getenv("LANGFUSE_PUBLIC_KEY", "your-public-key-here"), + "host": os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com"), + "use_model_name_instead_of_id_for_generation": os.getenv("USE_MODEL_NAME", "false").lower() == "true", + "debug": os.getenv("DEBUG_MODE", "false").lower() == "true", + } + ) + + self.langfuse = None + self.chat_traces = {} + self.suppressed_logs = set() + # Dictionary to store model names for each chat + self.model_names = {} + + # Only these tasks will be treated as LLM "generations": + self.GENERATION_TASKS = {"llm_response"} + + def log(self, message: str, suppress_repeats: bool = False): + if self.valves.debug: + if suppress_repeats: + if message in self.suppressed_logs: + return + self.suppressed_logs.add(message) + print(f"[DEBUG] {message}") + + async def on_startup(self): + self.log(f"on_startup triggered for {__name__}") + self.set_langfuse() + + async def on_shutdown(self): + self.log(f"on_shutdown triggered for {__name__}") + if self.langfuse: + self.langfuse.flush() + + async def on_valves_updated(self): + self.log("Valves updated, resetting Langfuse client.") + self.set_langfuse() + + def set_langfuse(self): + try: + self.langfuse = Langfuse( + secret_key=self.valves.secret_key, + public_key=self.valves.public_key, + host=self.valves.host, + debug=self.valves.debug, + ) + self.langfuse.auth_check() + self.log("Langfuse client initialized successfully.") + except UnauthorizedError: + print( + "Langfuse credentials incorrect. Please re-enter your Langfuse credentials in the pipeline settings." + ) + except Exception as e: + print( + f"Langfuse error: {e} Please re-enter your Langfuse credentials in the pipeline settings." + ) + + def _build_tags(self, task_name: str) -> list: + """ + Builds a list of tags based on valve settings, ensuring we always add + 'open-webui' and skip user_response / llm_response from becoming tags themselves. + """ + tags_list = [] + if self.valves.insert_tags: + # Always add 'open-webui' + tags_list.append("open-webui") + # Add the task_name if it's not one of the excluded defaults + if task_name not in ["user_response", "llm_response"]: + tags_list.append(task_name) + return tags_list + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + if self.valves.debug: + print(f"[DEBUG] Received request: {json.dumps(body, indent=2)}") + + self.log(f"Inlet function called with body: {body} and user: {user}") + + metadata = body.get("metadata", {}) + chat_id = metadata.get("chat_id", str(uuid.uuid4())) + + # Handle temporary chats + if chat_id == "local": + session_id = metadata.get("session_id") + chat_id = f"temporary-session-{session_id}" + + metadata["chat_id"] = chat_id + body["metadata"] = metadata + + # Extract and store both model name and ID if available + model_info = metadata.get("model", {}) + model_id = body.get("model") + + # Store model information for this chat + if chat_id not in self.model_names: + self.model_names[chat_id] = {"id": model_id} + else: + self.model_names[chat_id]["id"] = model_id + + if isinstance(model_info, dict) and "name" in model_info: + self.model_names[chat_id]["name"] = model_info["name"] + self.log(f"Stored model info - name: '{model_info['name']}', id: '{model_id}' for chat_id: {chat_id}") + + required_keys = ["model", "messages"] + missing_keys = [key for key in required_keys if key not in body] + if missing_keys: + error_message = f"Error: Missing keys in the request body: {', '.join(missing_keys)}" + self.log(error_message) + raise ValueError(error_message) + + user_email = user.get("email") if user else None + # Defaulting to 'user_response' if no task is provided + task_name = metadata.get("task", "user_response") + + # Build tags + tags_list = self._build_tags(task_name) + + if chat_id not in self.chat_traces: + self.log(f"Creating new trace for chat_id: {chat_id}") + + trace_payload = { + "name": f"chat:{chat_id}", + "input": body, + "user_id": user_email, + "metadata": metadata, + "session_id": chat_id, + } + + if tags_list: + trace_payload["tags"] = tags_list + + if self.valves.debug: + print(f"[DEBUG] Langfuse trace request: {json.dumps(trace_payload, indent=2)}") + + trace = self.langfuse.trace(**trace_payload) + self.chat_traces[chat_id] = trace + else: + trace = self.chat_traces[chat_id] + self.log(f"Reusing existing trace for chat_id: {chat_id}") + if tags_list: + trace.update(tags=tags_list) + + # Update metadata with type + metadata["type"] = task_name + metadata["interface"] = "open-webui" + + # If it's a task that is considered an LLM generation + if task_name in self.GENERATION_TASKS: + # Determine which model value to use based on the use_model_name valve + model_id = self.model_names.get(chat_id, {}).get("id", body["model"]) + model_name = self.model_names.get(chat_id, {}).get("name", "unknown") + + # Pick primary model identifier based on valve setting + model_value = model_name if self.valves.use_model_name_instead_of_id_for_generation else model_id + + # Add both values to metadata regardless of valve setting + metadata["model_id"] = model_id + metadata["model_name"] = model_name + + generation_payload = { + "name": f"{task_name}:{str(uuid.uuid4())}", + "model": model_value, + "input": body["messages"], + "metadata": metadata, + } + if tags_list: + generation_payload["tags"] = tags_list + + if self.valves.debug: + print(f"[DEBUG] Langfuse generation request: {json.dumps(generation_payload, indent=2)}") + + trace.generation(**generation_payload) + else: + # Otherwise, log it as an event + event_payload = { + "name": f"{task_name}:{str(uuid.uuid4())}", + "metadata": metadata, + "input": body["messages"], + } + if tags_list: + event_payload["tags"] = tags_list + + if self.valves.debug: + print(f"[DEBUG] Langfuse event request: {json.dumps(event_payload, indent=2)}") + + trace.event(**event_payload) + + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + self.log(f"Outlet function called with body: {body}") + + chat_id = body.get("chat_id") + + # Handle temporary chats + if chat_id == "local": + session_id = body.get("session_id") + chat_id = f"temporary-session-{session_id}" + + metadata = body.get("metadata", {}) + # Defaulting to 'llm_response' if no task is provided + task_name = metadata.get("task", "llm_response") + + # Build tags + tags_list = self._build_tags(task_name) + + if chat_id not in self.chat_traces: + self.log(f"[WARNING] No matching trace found for chat_id: {chat_id}, attempting to re-register.") + # Re-run inlet to register if somehow missing + return await self.inlet(body, user) + + trace = self.chat_traces[chat_id] + + assistant_message = get_last_assistant_message(body["messages"]) + assistant_message_obj = get_last_assistant_message_obj(body["messages"]) + + usage = None + if assistant_message_obj: + info = assistant_message_obj.get("usage", {}) + if isinstance(info, dict): + input_tokens = info.get("prompt_eval_count") or info.get("prompt_tokens") + output_tokens = info.get("eval_count") or info.get("completion_tokens") + if input_tokens is not None and output_tokens is not None: + usage = { + "input": input_tokens, + "output": output_tokens, + "unit": "TOKENS", + } + self.log(f"Usage data extracted: {usage}") + + # Update the trace output with the last assistant message + trace.update(output=assistant_message) + + metadata["type"] = task_name + metadata["interface"] = "open-webui" + + if task_name in self.GENERATION_TASKS: + # Determine which model value to use based on the use_model_name valve + model_id = self.model_names.get(chat_id, {}).get("id", body.get("model")) + model_name = self.model_names.get(chat_id, {}).get("name", "unknown") + + # Pick primary model identifier based on valve setting + model_value = model_name if self.valves.use_model_name_instead_of_id_for_generation else model_id + + # Add both values to metadata regardless of valve setting + metadata["model_id"] = model_id + metadata["model_name"] = model_name + + # If it's an LLM generation + generation_payload = { + "name": f"{task_name}:{str(uuid.uuid4())}", + "model": model_value, # <-- Use model name or ID based on valve setting + "input": body["messages"], + "metadata": metadata, + "usage": usage, + } + if tags_list: + generation_payload["tags"] = tags_list + + if self.valves.debug: + print(f"[DEBUG] Langfuse generation end request: {json.dumps(generation_payload, indent=2)}") + + trace.generation().end(**generation_payload) + self.log(f"Generation ended for chat_id: {chat_id}") + else: + # Otherwise log as an event + event_payload = { + "name": f"{task_name}:{str(uuid.uuid4())}", + "metadata": metadata, + "input": body["messages"], + } + if usage: + # If you want usage on event as well + event_payload["metadata"]["usage"] = usage + + if tags_list: + event_payload["tags"] = tags_list + + if self.valves.debug: + print(f"[DEBUG] Langfuse event end request: {json.dumps(event_payload, indent=2)}") + + trace.event(**event_payload) + self.log(f"Event logged for chat_id: {chat_id}") + + return body diff --git a/openwebui/pipelines/examples/filters/langfuse_v3_filter_pipeline.py b/openwebui/pipelines/examples/filters/langfuse_v3_filter_pipeline.py new file mode 100644 index 0000000..a046eee --- /dev/null +++ b/openwebui/pipelines/examples/filters/langfuse_v3_filter_pipeline.py @@ -0,0 +1,406 @@ +""" +title: Langfuse Filter Pipeline for v3 +author: open-webui +date: 2025-07-31 +version: 0.0.1 +license: MIT +description: A filter pipeline that uses Langfuse v3. +requirements: langfuse>=3.0.0 +""" + +from typing import List, Optional +import os +import uuid +import json + + +from utils.pipelines.main import get_last_assistant_message +from pydantic import BaseModel +from langfuse import Langfuse + + +def get_last_assistant_message_obj(messages: List[dict]) -> dict: + """Retrieve the last assistant message from the message list.""" + for message in reversed(messages): + if message["role"] == "assistant": + return message + return {} + + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + secret_key: str + public_key: str + host: str + # New valve that controls whether task names are added as tags: + insert_tags: bool = True + # New valve that controls whether to use model name instead of model ID for generation + use_model_name_instead_of_id_for_generation: bool = False + debug: bool = False + + def __init__(self): + self.type = "filter" + self.name = "Langfuse Filter" + + self.valves = self.Valves( + **{ + "pipelines": ["*"], + "secret_key": os.getenv("LANGFUSE_SECRET_KEY", "your-secret-key-here"), + "public_key": os.getenv("LANGFUSE_PUBLIC_KEY", "your-public-key-here"), + "host": os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com"), + "use_model_name_instead_of_id_for_generation": os.getenv("USE_MODEL_NAME", "false").lower() == "true", + "debug": os.getenv("DEBUG_MODE", "false").lower() == "true", + } + ) + + self.langfuse = None + self.chat_traces = {} + self.suppressed_logs = set() + # Dictionary to store model names for each chat + self.model_names = {} + + def log(self, message: str, suppress_repeats: bool = False): + if self.valves.debug: + if suppress_repeats: + if message in self.suppressed_logs: + return + self.suppressed_logs.add(message) + print(f"[DEBUG] {message}") + + async def on_startup(self): + self.log(f"on_startup triggered for {__name__}") + self.set_langfuse() + + async def on_shutdown(self): + self.log(f"on_shutdown triggered for {__name__}") + if self.langfuse: + try: + # End all active traces + for chat_id, trace in self.chat_traces.items(): + try: + trace.end() + self.log(f"Ended trace for chat_id: {chat_id}") + except Exception as e: + self.log(f"Failed to end trace for {chat_id}: {e}") + + self.chat_traces.clear() + self.langfuse.flush() + self.log("Langfuse data flushed on shutdown") + except Exception as e: + self.log(f"Failed to flush Langfuse data: {e}") + + async def on_valves_updated(self): + self.log("Valves updated, resetting Langfuse client.") + self.set_langfuse() + + def set_langfuse(self): + try: + self.log(f"Initializing Langfuse with host: {self.valves.host}") + self.log( + f"Secret key set: {'Yes' if self.valves.secret_key and self.valves.secret_key != 'your-secret-key-here' else 'No'}" + ) + self.log( + f"Public key set: {'Yes' if self.valves.public_key and self.valves.public_key != 'your-public-key-here' else 'No'}" + ) + + # Initialize Langfuse client for v3.2.1 + self.langfuse = Langfuse( + secret_key=self.valves.secret_key, + public_key=self.valves.public_key, + host=self.valves.host, + debug=self.valves.debug, + ) + + # Test authentication + try: + self.langfuse.auth_check() + self.log( + f"Langfuse client initialized and authenticated successfully. Connected to host: {self.valves.host}") + + except Exception as e: + self.log(f"Auth check failed: {e}") + self.log(f"Failed host: {self.valves.host}") + self.langfuse = None + return + + except Exception as auth_error: + if ( + "401" in str(auth_error) + or "unauthorized" in str(auth_error).lower() + or "credentials" in str(auth_error).lower() + ): + self.log(f"Langfuse credentials incorrect: {auth_error}") + self.langfuse = None + return + except Exception as e: + self.log(f"Langfuse initialization error: {e}") + self.langfuse = None + + def _build_tags(self, task_name: str) -> list: + """ + Builds a list of tags based on valve settings, ensuring we always add + 'open-webui' and skip user_response / llm_response from becoming tags themselves. + """ + tags_list = [] + if self.valves.insert_tags: + # Always add 'open-webui' + tags_list.append("open-webui") + # Add the task_name if it's not one of the excluded defaults + if task_name not in ["user_response", "llm_response"]: + tags_list.append(task_name) + return tags_list + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + self.log("Langfuse Filter INLET called") + + # Check Langfuse client status + if not self.langfuse: + self.log("[WARNING] Langfuse client not initialized - Skipped") + return body + + self.log(f"Inlet function called with body: {body} and user: {user}") + + metadata = body.get("metadata", {}) + chat_id = metadata.get("chat_id", str(uuid.uuid4())) + + # Handle temporary chats + if chat_id == "local": + session_id = metadata.get("session_id") + chat_id = f"temporary-session-{session_id}" + + metadata["chat_id"] = chat_id + body["metadata"] = metadata + + # Extract and store both model name and ID if available + model_info = metadata.get("model", {}) + model_id = body.get("model") + + # Store model information for this chat + if chat_id not in self.model_names: + self.model_names[chat_id] = {"id": model_id} + else: + self.model_names[chat_id]["id"] = model_id + + if isinstance(model_info, dict) and "name" in model_info: + self.model_names[chat_id]["name"] = model_info["name"] + self.log(f"Stored model info - name: '{model_info['name']}', id: '{model_id}' for chat_id: {chat_id}") + + required_keys = ["model", "messages"] + missing_keys = [key for key in required_keys if key not in body] + if missing_keys: + error_message = f"Error: Missing keys in the request body: {', '.join(missing_keys)}" + self.log(error_message) + raise ValueError(error_message) + + user_email = user.get("email") if user else None + # Defaulting to 'user_response' if no task is provided + task_name = metadata.get("task", "user_response") + + # Build tags + tags_list = self._build_tags(task_name) + + if chat_id not in self.chat_traces: + self.log(f"Creating new trace for chat_id: {chat_id}") + + try: + # Create trace using Langfuse v3 API with complete data + trace_metadata = { + **metadata, + "user_id": user_email, + "session_id": chat_id, + "interface": "open-webui", + } + + # Create trace with all necessary information + trace = self.langfuse.start_span( + name=f"chat:{chat_id}", + input=body, + metadata=trace_metadata + ) + + # Set additional trace attributes + trace.update_trace( + user_id=user_email, + session_id=chat_id, + tags=tags_list if tags_list else None, + input=body, + metadata=trace_metadata, + ) + + self.chat_traces[chat_id] = trace + self.log(f"Successfully created trace for chat_id: {chat_id}") + except Exception as e: + self.log(f"Failed to create trace: {e}") + return body + else: + trace = self.chat_traces[chat_id] + self.log(f"Reusing existing trace for chat_id: {chat_id}") + # Update trace with current metadata and tags + trace_metadata = { + **metadata, + "user_id": user_email, + "session_id": chat_id, + "interface": "open-webui", + } + trace.update_trace( + tags=tags_list if tags_list else None, + metadata=trace_metadata, + ) + + # Update metadata with type + metadata["type"] = task_name + metadata["interface"] = "open-webui" + + # Log user input as event + try: + trace = self.chat_traces[chat_id] + + # Create complete event metadata + event_metadata = { + **metadata, + "type": "user_input", + "interface": "open-webui", + "user_id": user_email, + "session_id": chat_id, + "event_id": str(uuid.uuid4()), + } + + event_span = trace.start_span( + name=f"user_input:{str(uuid.uuid4())}", + metadata=event_metadata, + input=body["messages"], + ) + event_span.end() + self.log(f"User input event logged for chat_id: {chat_id}") + except Exception as e: + self.log(f"Failed to log user input event: {e}") + + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + self.log("Langfuse Filter OUTLET called") + + # Check Langfuse client status + if not self.langfuse: + self.log("[WARNING] Langfuse client not initialized - Skipped") + return body + + self.log(f"Outlet function called with body: {body}") + + chat_id = body.get("chat_id") + + # Handle temporary chats + if chat_id == "local": + session_id = body.get("session_id") + chat_id = f"temporary-session-{session_id}" + + metadata = body.get("metadata", {}) + # Defaulting to 'llm_response' if no task is provided + task_name = metadata.get("task", "llm_response") + + # Build tags + tags_list = self._build_tags(task_name) + + if chat_id not in self.chat_traces: + self.log(f"[WARNING] No matching trace found for chat_id: {chat_id}, attempting to re-register.") + # Re-run inlet to register if somehow missing + return await self.inlet(body, user) + + self.chat_traces[chat_id] + + assistant_message = get_last_assistant_message(body["messages"]) + assistant_message_obj = get_last_assistant_message_obj(body["messages"]) + + usage = None + if assistant_message_obj: + info = assistant_message_obj.get("usage", {}) + if isinstance(info, dict): + input_tokens = info.get("prompt_eval_count") or info.get("prompt_tokens") + output_tokens = info.get("eval_count") or info.get("completion_tokens") + if input_tokens is not None and output_tokens is not None: + usage = { + "input": input_tokens, + "output": output_tokens, + "unit": "TOKENS", + } + self.log(f"Usage data extracted: {usage}") + + # Update the trace with complete output information + trace = self.chat_traces[chat_id] + + metadata["type"] = task_name + metadata["interface"] = "open-webui" + + # Create complete trace metadata with all information + complete_trace_metadata = { + **metadata, + "user_id": user.get("email") if user else None, + "session_id": chat_id, + "interface": "open-webui", + "task": task_name, + } + + # Update trace with output and complete metadata + trace.update_trace( + output=assistant_message, + metadata=complete_trace_metadata, + tags=tags_list if tags_list else None, + ) + + # Outlet: Always create LLM generation (this is the LLM response) + # Determine which model value to use based on the use_model_name valve + model_id = self.model_names.get(chat_id, {}).get("id", body.get("model")) + model_name = self.model_names.get(chat_id, {}).get("name", "unknown") + + # Pick primary model identifier based on valve setting + model_value = ( + model_name + if self.valves.use_model_name_instead_of_id_for_generation + else model_id + ) + + # Add both values to metadata regardless of valve setting + metadata["model_id"] = model_id + metadata["model_name"] = model_name + + # Create LLM generation for the response + try: + trace = self.chat_traces[chat_id] + + # Create complete generation metadata + generation_metadata = { + **complete_trace_metadata, + "type": "llm_response", + "model_id": model_id, + "model_name": model_name, + "generation_id": str(uuid.uuid4()), + } + + generation = trace.start_generation( + name=f"llm_response:{str(uuid.uuid4())}", + model=model_value, + input=body["messages"], + output=assistant_message, + metadata=generation_metadata, + ) + + # Update with usage if available + if usage: + generation.update(usage=usage) + + generation.end() + self.log(f"LLM generation completed for chat_id: {chat_id}") + except Exception as e: + self.log(f"Failed to create LLM generation: {e}") + + # Flush data to Langfuse + try: + if self.langfuse: + self.langfuse.flush() + self.log("Langfuse data flushed") + except Exception as e: + self.log(f"Failed to flush Langfuse data: {e}") + + return body diff --git a/openwebui/pipelines/examples/filters/libretranslate_filter_pipeline.py b/openwebui/pipelines/examples/filters/libretranslate_filter_pipeline.py new file mode 100644 index 0000000..39e8c5f --- /dev/null +++ b/openwebui/pipelines/examples/filters/libretranslate_filter_pipeline.py @@ -0,0 +1,141 @@ +from typing import List, Optional +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import os + +from utils.pipelines.main import get_last_user_message, get_last_assistant_message + + +class Pipeline: + + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + # e.g. ["llama3:latest", "gpt-3.5-turbo"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Valves + libretranslate_url: str + + # Source and target languages + # User message will be translated from source_user to target_user + source_user: Optional[str] = "auto" + target_user: Optional[str] = "en" + + # Assistant languages + # Assistant message will be translated from source_assistant to target_assistant + source_assistant: Optional[str] = "en" + target_assistant: Optional[str] = "es" + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "libretranslate_filter_pipeline" + self.name = "LibreTranslate Filter" + + # Initialize + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + "libretranslate_url": os.getenv( + "LIBRETRANSLATE_API_BASE_URL", "http://localhost:5000" + ), + } + ) + + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + pass + + def translate(self, text: str, source: str, target: str) -> str: + payload = { + "q": text, + "source": source, + "target": target, + } + + try: + r = requests.post( + f"{self.valves.libretranslate_url}/translate", json=payload + ) + r.raise_for_status() + + data = r.json() + return data["translatedText"] + except Exception as e: + print(f"Error translating text: {e}") + return text + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"inlet:{__name__}") + + messages = body["messages"] + user_message = get_last_user_message(messages) + + print(f"User message: {user_message}") + + # Translate user message + translated_user_message = self.translate( + user_message, + self.valves.source_user, + self.valves.target_user, + ) + + print(f"Translated user message: {translated_user_message}") + + for message in reversed(messages): + if message["role"] == "user": + message["content"] = translated_user_message + break + + body = {**body, "messages": messages} + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"outlet:{__name__}") + + messages = body["messages"] + assistant_message = get_last_assistant_message(messages) + + print(f"Assistant message: {assistant_message}") + + # Translate assistant message + translated_assistant_message = self.translate( + assistant_message, + self.valves.source_assistant, + self.valves.target_assistant, + ) + + print(f"Translated assistant message: {translated_assistant_message}") + + for message in reversed(messages): + if message["role"] == "assistant": + message["content"] = translated_assistant_message + break + + body = {**body, "messages": messages} + return body diff --git a/openwebui/pipelines/examples/filters/llm_translate_filter_pipeline.py b/openwebui/pipelines/examples/filters/llm_translate_filter_pipeline.py new file mode 100644 index 0000000..a97067b --- /dev/null +++ b/openwebui/pipelines/examples/filters/llm_translate_filter_pipeline.py @@ -0,0 +1,157 @@ +from typing import List, Optional +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import os + +from utils.pipelines.main import get_last_user_message, get_last_assistant_message + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + # e.g. ["llama3:latest", "gpt-3.5-turbo"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + OPENAI_API_BASE_URL: str = "https://api.openai.com/v1" + OPENAI_API_KEY: str = "" + TASK_MODEL: str = "gpt-3.5-turbo" + + # Source and target languages + # User message will be translated from source_user to target_user + source_user: Optional[str] = "auto" + target_user: Optional[str] = "en" + + # Assistant languages + # Assistant message will be translated from source_assistant to target_assistant + source_assistant: Optional[str] = "en" + target_assistant: Optional[str] = "es" + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "libretranslate_filter_pipeline" + self.name = "LLM Translate Filter" + + # Initialize + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + "OPENAI_API_KEY": os.getenv( + "OPENAI_API_KEY", "your-openai-api-key-here" + ), + } + ) + + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + pass + + def translate(self, text: str, source: str, target: str) -> str: + headers = {} + headers["Authorization"] = f"Bearer {self.valves.OPENAI_API_KEY}" + headers["Content-Type"] = "application/json" + + payload = { + "messages": [ + { + "role": "system", + "content": f"Translate the following text to {target}. Provide only the translated text and nothing else.", + }, + {"role": "user", "content": text}, + ], + "model": self.valves.TASK_MODEL, + } + print(payload) + + try: + r = requests.post( + url=f"{self.valves.OPENAI_API_BASE_URL}/chat/completions", + json=payload, + headers=headers, + stream=False, + ) + + r.raise_for_status() + response = r.json() + print(response) + return response["choices"][0]["message"]["content"] + except Exception as e: + return f"Error: {e}" + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"inlet:{__name__}") + + messages = body["messages"] + user_message = get_last_user_message(messages) + + print(f"User message: {user_message}") + + # Translate user message + translated_user_message = self.translate( + user_message, + self.valves.source_user, + self.valves.target_user, + ) + + print(f"Translated user message: {translated_user_message}") + + for message in reversed(messages): + if message["role"] == "user": + message["content"] = translated_user_message + break + + body = {**body, "messages": messages} + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + if "title" in body: + return body + + print(f"outlet:{__name__}") + + messages = body["messages"] + assistant_message = get_last_assistant_message(messages) + + print(f"Assistant message: {assistant_message}") + + # Translate assistant message + translated_assistant_message = self.translate( + assistant_message, + self.valves.source_assistant, + self.valves.target_assistant, + ) + + print(f"Translated assistant message: {translated_assistant_message}") + + for message in reversed(messages): + if message["role"] == "assistant": + message["content"] = translated_assistant_message + break + + body = {**body, "messages": messages} + return body diff --git a/openwebui/pipelines/examples/filters/llmguard_prompt_injection_filter_pipeline.py b/openwebui/pipelines/examples/filters/llmguard_prompt_injection_filter_pipeline.py new file mode 100644 index 0000000..b3cd79e --- /dev/null +++ b/openwebui/pipelines/examples/filters/llmguard_prompt_injection_filter_pipeline.py @@ -0,0 +1,81 @@ +""" +title: LLM Guard Filter Pipeline +author: jannikstdl +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for filtering out potential prompt injections using the LLM Guard library. +requirements: llm-guard +""" + +from typing import List, Optional +from schemas import OpenAIChatMessage +from pydantic import BaseModel +from llm_guard.input_scanners import PromptInjection +from llm_guard.input_scanners.prompt_injection import MatchType +import os + +class Pipeline: + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Assign a unique identifier to the pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + self.id = "llmguard_prompt_injection_filter_pipeline" + self.name = "LLMGuard Prompt Injection Filter" + + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + # e.g. ["llama3:latest", "gpt-3.5-turbo"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Initialize + self.valves = Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + } + ) + + self.model = None + + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + + self.model = PromptInjection(threshold=0.8, match_type=MatchType.FULL) + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This filter is applied to the form data before it is sent to the OpenAI API. + print(f"inlet:{__name__}") + + user_message = body["messages"][-1]["content"] + + # Filter out prompt injection messages + sanitized_prompt, is_valid, risk_score = self.model.scan(user_message) + + if risk_score > 0.8: + raise Exception("Prompt injection detected") + + return body diff --git a/openwebui/pipelines/examples/filters/mem0_memory_filter_pipeline.py b/openwebui/pipelines/examples/filters/mem0_memory_filter_pipeline.py new file mode 100644 index 0000000..2016c61 --- /dev/null +++ b/openwebui/pipelines/examples/filters/mem0_memory_filter_pipeline.py @@ -0,0 +1,140 @@ +""" +title: Long Term Memory Filter +author: Anton Nilsson +date: 2024-08-23 +version: 1.0 +license: MIT +description: A filter that processes user messages and stores them as long term memory by utilizing the mem0 framework together with qdrant and ollama +requirements: pydantic, ollama, mem0ai +""" + +from typing import List, Optional +from pydantic import BaseModel +import json +from mem0 import Memory +import threading + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + + store_cycles: int = 5 # Number of messages from the user before the data is processed and added to the memory + mem_zero_user: str = "user" # Memories belongs to this user, only used by mem0 for internal organization of memories + + # Default values for the mem0 vector store + vector_store_qdrant_name: str = "memories" + vector_store_qdrant_url: str = "host.docker.internal" + vector_store_qdrant_port: int = 6333 + vector_store_qdrant_dims: int = 768 # Need to match the vector dimensions of the embedder model + + # Default values for the mem0 language model + ollama_llm_model: str = "llama3.1:latest" # This model need to exist in ollama + ollama_llm_temperature: float = 0 + ollama_llm_tokens: int = 8000 + ollama_llm_url: str = "http://host.docker.internal:11434" + + # Default values for the mem0 embedding model + ollama_embedder_model: str = "nomic-embed-text:latest" # This model need to exist in ollama + ollama_embedder_url: str = "http://host.docker.internal:11434" + + def __init__(self): + self.type = "filter" + self.name = "Memory Filter" + self.user_messages = [] + self.thread = None + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + } + ) + self.m = self.init_mem_zero() + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"pipe:{__name__}") + + user = self.valves.mem_zero_user + store_cycles = self.valves.store_cycles + + if isinstance(body, str): + body = json.loads(body) + + all_messages = body["messages"] + last_message = all_messages[-1]["content"] + + self.user_messages.append(last_message) + + if len(self.user_messages) == store_cycles: + + message_text = "" + for message in self.user_messages: + message_text += message + " " + + if self.thread and self.thread.is_alive(): + print("Waiting for previous memory to be done") + self.thread.join() + + self.thread = threading.Thread(target=self.m.add, kwargs={"data":message_text,"user_id":user}) + + print("Text to be processed in to a memory:") + print(message_text) + + self.thread.start() + self.user_messages.clear() + + memories = self.m.search(last_message, user_id=user) + + if(memories): + fetched_memory = memories[0]["memory"] + else: + fetched_memory = "" + + print("Memory added to the context:") + print(fetched_memory) + + if fetched_memory: + all_messages.insert(0, {"role":"system", "content":"This is your inner voice talking, you remember this about the person you chatting with "+str(fetched_memory)}) + + print("Final body to send to the LLM:") + print(body) + + return body + + def init_mem_zero(self): + config = { + "vector_store": { + "provider": "qdrant", + "config": { + "collection_name": self.valves.vector_store_qdrant_name, + "host": self.valves.vector_store_qdrant_url, + "port": self.valves.vector_store_qdrant_port, + "embedding_model_dims": self.valves.vector_store_qdrant_dims, + }, + }, + "llm": { + "provider": "ollama", + "config": { + "model": self.valves.ollama_llm_model, + "temperature": self.valves.ollama_llm_temperature, + "max_tokens": self.valves.ollama_llm_tokens, + "ollama_base_url": self.valves.ollama_llm_url, + }, + }, + "embedder": { + "provider": "ollama", + "config": { + "model": self.valves.ollama_embedder_model, + "ollama_base_url": self.valves.ollama_embedder_url, + }, + }, + } + + return Memory.from_config(config) \ No newline at end of file diff --git a/openwebui/pipelines/examples/filters/opik_filter_pipeline.py b/openwebui/pipelines/examples/filters/opik_filter_pipeline.py new file mode 100644 index 0000000..ab768b0 --- /dev/null +++ b/openwebui/pipelines/examples/filters/opik_filter_pipeline.py @@ -0,0 +1,274 @@ +""" +title: Opik Filter Pipeline +author: open-webui +date: 2025-03-12 +version: 1.0 +license: MIT +description: A filter pipeline that uses Opik for LLM observability. +requirements: opik +""" + +from typing import List, Optional +import os +import uuid +import json + +from pydantic import BaseModel +from opik import Opik + + +def get_last_assistant_message_obj(messages: List[dict]) -> dict: + for message in reversed(messages): + if message["role"] == "assistant": + return message + return {} + + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = [] + priority: int = 0 + api_key: Optional[str] = None + workspace: str + project_name: str + host: str + debug: bool = False + + def __init__(self): + self.type = "filter" + self.name = "Opik Filter" + + self.valves = self.Valves( + **{ + "pipelines": ["*"], + "api_key": os.getenv("OPIK_API_KEY", "set_me_for_opik_cloud"), + "workspace": os.getenv("OPIK_WORKSPACE", "default"), + "project_name": os.getenv("OPIK_PROJECT_NAME", "default"), + "host": os.getenv( + "OPIK_URL_OVERRIDE", "https://www.comet.com/opik/api" + ), + "debug": os.getenv("DEBUG_MODE", "false").lower() == "true", + } + ) + + self.opik = None + # Keep track of the trace and the last-created span for each chat_id + self.chat_traces = {} + self.chat_spans = {} + + self.suppressed_logs = set() + + def log(self, message: str, suppress_repeats: bool = False): + """Logs messages to the terminal if debugging is enabled.""" + if self.valves.debug: + if suppress_repeats: + if message in self.suppressed_logs: + return + self.suppressed_logs.add(message) + print(f"[DEBUG] {message}") + + async def on_startup(self): + self.log(f"on_startup triggered for {__name__}") + self.set_opik() + + async def on_shutdown(self): + self.log(f"on_shutdown triggered for {__name__}") + if self.opik: + self.opik.end() + + async def on_valves_updated(self): + self.log("Valves updated, resetting Opik client.") + if self.opik: + self.opik.end() + self.set_opik() + + def set_opik(self): + try: + self.opik = Opik( + project_name=self.valves.project_name, + workspace=self.valves.workspace, + host=self.valves.host, + api_key=self.valves.api_key, + ) + self.opik.auth_check() + self.log("Opik client initialized successfully.") + except Exception as e: + print( + f"Opik error: {e} Please re-enter your Opik credentials in the pipeline settings." + ) + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + """ + Inlet handles the incoming request (usually a user message). + - If no trace exists yet for this chat_id, we create a new trace. + - If a trace does exist, we simply create a new span for the new user message. + """ + if self.valves.debug: + print(f"[DEBUG] Received request: {json.dumps(body, indent=2)}") + + self.log(f"Inlet function called with body: {body} and user: {user}") + + metadata = body.get("metadata", {}) + task = metadata.get("task", "") + + # Skip logging tasks for now + if task: + self.log(f"Skipping {task} task.") + return body + + if "chat_id" not in metadata: + chat_id = str(uuid.uuid4()) # Regular chat messages + self.log(f"Assigned normal chat_id: {chat_id}") + + metadata["chat_id"] = chat_id + body["metadata"] = metadata + else: + chat_id = metadata["chat_id"] + + required_keys = ["model", "messages"] + missing_keys = [key for key in required_keys if key not in body] + if missing_keys: + error_message = ( + f"Error: Missing keys in the request body: {', '.join(missing_keys)}" + ) + self.log(error_message) + raise ValueError(error_message) + + user_email = user.get("email") if user else None + + assert chat_id not in self.chat_traces, ( + f"There shouldn't be a trace already exists for chat_id {chat_id}" + ) + + # Create a new trace and span + self.log(f"Creating new chat trace for chat_id: {chat_id}") + + # Body copy for traces and span + trace_body = body.copy() + span_body = body.copy() + + # Extract metadata from body + metadata = trace_body.pop("metadata", {}) + metadata.update({"chat_id": chat_id, "user_id": user_email}) + + # We don't need the model at the trace level + trace_body.pop("model", None) + + trace_payload = { + "name": f"{__name__}", + "input": trace_body, + "metadata": metadata, + "thread_id": chat_id, + } + + if self.valves.debug: + print(f"[DEBUG] Opik trace request: {json.dumps(trace_payload, indent=2)}") + + trace = self.opik.trace(**trace_payload) + + span_metadata = metadata.copy() + span_metadata.update({"interface": "open-webui"}) + + # Extract the model from body + span_body.pop("model", None) + # We don't need the metadata in the input for the span + span_body.pop("metadata", None) + + # Extract the model and provider from metadata + model = span_metadata.get("model", {}).get("id", None) + provider = span_metadata.get("model", {}).get("owned_by", None) + + span_payload = { + "name": chat_id, + "model": model, + "provider": provider, + "input": span_body, + "metadata": span_metadata, + "type": "llm", + } + + if self.valves.debug: + print(f"[DEBUG] Opik span request: {json.dumps(span_payload, indent=2)}") + + span = trace.span(**span_payload) + + self.chat_traces[chat_id] = trace + self.chat_spans[chat_id] = span + self.log(f"Trace and span objects successfully created for chat_id: {chat_id}") + + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + """ + Outlet handles the response body (usually the assistant message). + It will finalize/end the span created for the user request. + """ + self.log(f"Outlet function called with body: {body}") + + chat_id = body.get("chat_id") + + # If no trace or span exist, attempt to register again + if chat_id not in self.chat_traces or chat_id not in self.chat_spans: + self.log( + f"[WARNING] No matching chat trace found for chat_id: {chat_id}, chat won't be logged." + ) + return body + + trace = self.chat_traces[chat_id] + span = self.chat_spans[chat_id] + + # Body copy for traces and span + trace_body = body.copy() + span_body = body.copy() + + # Get the last assistant message from the conversation + assistant_message_obj = get_last_assistant_message_obj(body["messages"]) + + # Extract usage if available + usage = None + self.log(f"Assistant message obj: {assistant_message_obj}") + if assistant_message_obj: + message_usage = assistant_message_obj.get("usage", {}) + if isinstance(message_usage, dict): + input_tokens = message_usage.get( + "prompt_eval_count" + ) or message_usage.get("prompt_tokens") + output_tokens = message_usage.get("eval_count") or message_usage.get( + "completion_tokens" + ) + if input_tokens is not None and output_tokens is not None: + usage = { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + self.log(f"Usage data extracted: {usage}") + + # Chat_id is already logged as trace thread + span_body.pop("chat_id", None) + + # End the span with the final assistant message and updated conversation + span_payload = { + "output": span_body, # include the entire conversation + "usage": usage, + } + + if self.valves.debug: + print( + f"[DEBUG] Opik span end request: {json.dumps(span_payload, indent=2)}" + ) + + span.end(**span_payload) + self.log(f"span ended for chat_id: {chat_id}") + + # Chat_id is already logged as trace thread + span_body.pop("chat_id", None) + + # Optionally update the trace with the final assistant output + trace.end(output=trace_body) + + # Force the creation of a new trace and span for the next chat even if they are part of the same thread + del self.chat_traces[chat_id] + del self.chat_spans[chat_id] + + return body diff --git a/openwebui/pipelines/examples/filters/presidio_filter_pipeline.py b/openwebui/pipelines/examples/filters/presidio_filter_pipeline.py new file mode 100644 index 0000000..cca242d --- /dev/null +++ b/openwebui/pipelines/examples/filters/presidio_filter_pipeline.py @@ -0,0 +1,81 @@ +""" +title: Presidio PII Redaction Pipeline +author: justinh-rahb +date: 2024-07-07 +version: 0.1.0 +license: MIT +description: A pipeline for redacting personally identifiable information (PII) using the Presidio library. +requirements: presidio-analyzer, presidio-anonymizer +""" + +import os +from typing import List, Optional +from pydantic import BaseModel +from schemas import OpenAIChatMessage +from presidio_analyzer import AnalyzerEngine +from presidio_anonymizer import AnonymizerEngine +from presidio_anonymizer.entities import OperatorConfig + +class Pipeline: + class Valves(BaseModel): + pipelines: List[str] = ["*"] + priority: int = 0 + enabled_for_admins: bool = False + entities_to_redact: List[str] = [ + "PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER", "US_SSN", + "CREDIT_CARD", "IP_ADDRESS", "US_PASSPORT", "LOCATION", + "DATE_TIME", "NRP", "MEDICAL_LICENSE", "URL" + ] + language: str = "en" + + def __init__(self): + self.type = "filter" + self.name = "Presidio PII Redaction Pipeline" + + self.valves = self.Valves( + **{ + "pipelines": os.getenv("PII_REDACT_PIPELINES", "*").split(","), + "enabled_for_admins": os.getenv("PII_REDACT_ENABLED_FOR_ADMINS", "false").lower() == "true", + "entities_to_redact": os.getenv("PII_REDACT_ENTITIES", ",".join(self.Valves().entities_to_redact)).split(","), + "language": os.getenv("PII_REDACT_LANGUAGE", "en"), + } + ) + + self.analyzer = AnalyzerEngine() + self.anonymizer = AnonymizerEngine() + + async def on_startup(self): + print(f"on_startup:{__name__}") + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + + def redact_pii(self, text: str) -> str: + results = self.analyzer.analyze( + text=text, + language=self.valves.language, + entities=self.valves.entities_to_redact + ) + + anonymized_text = self.anonymizer.anonymize( + text=text, + analyzer_results=results, + operators={ + "DEFAULT": OperatorConfig("replace", {"new_value": "[REDACTED]"}) + } + ) + + return anonymized_text.text + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"pipe:{__name__}") + print(body) + print(user) + + if user is None or user.get("role") != "admin" or self.valves.enabled_for_admins: + messages = body.get("messages", []) + for message in messages: + if message.get("role") == "user": + message["content"] = self.redact_pii(message["content"]) + + return body diff --git a/openwebui/pipelines/examples/filters/rate_limit_filter_pipeline.py b/openwebui/pipelines/examples/filters/rate_limit_filter_pipeline.py new file mode 100644 index 0000000..d1e8823 --- /dev/null +++ b/openwebui/pipelines/examples/filters/rate_limit_filter_pipeline.py @@ -0,0 +1,127 @@ +import os +from typing import List, Optional +from pydantic import BaseModel +from schemas import OpenAIChatMessage +import time + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Valves for rate limiting + requests_per_minute: Optional[int] = None + requests_per_hour: Optional[int] = None + sliding_window_limit: Optional[int] = None + sliding_window_minutes: Optional[int] = None + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "rate_limit_filter_pipeline" + self.name = "Rate Limit Filter" + + # Initialize rate limits + self.valves = self.Valves( + **{ + "pipelines": os.getenv("RATE_LIMIT_PIPELINES", "*").split(","), + "requests_per_minute": int( + os.getenv("RATE_LIMIT_REQUESTS_PER_MINUTE", 10) + ), + "requests_per_hour": int( + os.getenv("RATE_LIMIT_REQUESTS_PER_HOUR", 1000) + ), + "sliding_window_limit": int( + os.getenv("RATE_LIMIT_SLIDING_WINDOW_LIMIT", 100) + ), + "sliding_window_minutes": int( + os.getenv("RATE_LIMIT_SLIDING_WINDOW_MINUTES", 15) + ), + } + ) + + # Tracking data - user_id -> (timestamps of requests) + self.user_requests = {} + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def prune_requests(self, user_id: str): + """Prune old requests that are outside of the sliding window period.""" + now = time.time() + if user_id in self.user_requests: + self.user_requests[user_id] = [ + req + for req in self.user_requests[user_id] + if ( + (self.valves.requests_per_minute is not None and now - req < 60) + or (self.valves.requests_per_hour is not None and now - req < 3600) + or ( + self.valves.sliding_window_limit is not None + and now - req < self.valves.sliding_window_minutes * 60 + ) + ) + ] + + def log_request(self, user_id: str): + """Log a new request for a user.""" + now = time.time() + if user_id not in self.user_requests: + self.user_requests[user_id] = [] + self.user_requests[user_id].append(now) + + def rate_limited(self, user_id: str) -> bool: + """Check if a user is rate limited.""" + self.prune_requests(user_id) + + user_reqs = self.user_requests.get(user_id, []) + + if self.valves.requests_per_minute is not None: + requests_last_minute = sum(1 for req in user_reqs if time.time() - req < 60) + if requests_last_minute >= self.valves.requests_per_minute: + return True + + if self.valves.requests_per_hour is not None: + requests_last_hour = sum(1 for req in user_reqs if time.time() - req < 3600) + if requests_last_hour >= self.valves.requests_per_hour: + return True + + if self.valves.sliding_window_limit is not None: + requests_in_window = len(user_reqs) + if requests_in_window >= self.valves.sliding_window_limit: + return True + + return False + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + print(f"pipe:{__name__}") + print(body) + print(user) + + if user.get("role", "admin") == "user": + user_id = user["id"] if user and "id" in user else "default_user" + if self.rate_limited(user_id): + raise Exception("Rate limit exceeded. Please try again later.") + + self.log_request(user_id) + return body diff --git a/openwebui/pipelines/examples/pipelines/events_pipeline.py b/openwebui/pipelines/examples/pipelines/events_pipeline.py new file mode 100644 index 0000000..2baece9 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/events_pipeline.py @@ -0,0 +1,83 @@ +from typing import List, Union, Generator, Iterator, Optional +from pprint import pprint +import time + +# Uncomment to disable SSL verification warnings if needed. +# warnings.filterwarnings('ignore', message='Unverified HTTPS request') + + +class Pipeline: + def __init__(self): + self.name = "Pipeline with Status Event" + self.description = ( + "This is a pipeline that demonstrates how to use the status event." + ) + self.debug = True + self.version = "0.1.0" + self.author = "Anthony Durussel" + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup: {__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is shutdown. + print(f"on_shutdown: {__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called before the OpenAI API request is made. You can modify the form data before it is sent to the OpenAI API. + print(f"inlet: {__name__}") + if self.debug: + print(f"inlet: {__name__} - body:") + pprint(body) + print(f"inlet: {__name__} - user:") + pprint(user) + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called after the OpenAI API response is completed. You can modify the messages after they are received from the OpenAI API. + print(f"outlet: {__name__}") + if self.debug: + print(f"outlet: {__name__} - body:") + pprint(body) + print(f"outlet: {__name__} - user:") + pprint(user) + return body + + def pipe( + self, + user_message: str, + model_id: str, + messages: List[dict], + body: dict, + ) -> Union[str, Generator, Iterator]: + print(f"pipe: {__name__}") + + if self.debug: + print(f"pipe: {__name__} - received message from user: {user_message}") + + yield { + "event": { + "type": "status", + "data": { + "description": "Fake Status", + "done": False, + }, + } + } + + time.sleep(5) # Sleep for 5 seconds + + yield f"user_message: {user_message}" + + yield { + "event": { + "type": "status", + "data": { + "description": "", + "done": True, + }, + } + } diff --git a/openwebui/pipelines/examples/pipelines/integrations/applescript_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/applescript_pipeline.py new file mode 100644 index 0000000..bc6266f --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/applescript_pipeline.py @@ -0,0 +1,89 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import requests + + +from subprocess import call + + +class Pipeline: + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "applescript_pipeline" + self.name = "AppleScript Pipeline" + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + OLLAMA_BASE_URL = "http://localhost:11434" + MODEL = "llama3" + + if body.get("title", False): + print("Title Generation") + return "AppleScript Pipeline" + else: + if "user" in body: + print("######################################") + print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})') + print(f"# Message: {user_message}") + print("######################################") + + commands = user_message.split(" ") + + if commands[0] == "volume": + + try: + commands[1] = int(commands[1]) + if 0 <= commands[1] <= 100: + call( + [f"osascript -e 'set volume output volume {commands[1]}'"], + shell=True, + ) + except: + pass + + payload = { + "model": MODEL, + "messages": [ + { + "role": "system", + "content": f"You are an agent of the AppleScript Pipeline. You have the power to control the volume of the system.", + }, + {"role": "user", "content": user_message}, + ], + "stream": body["stream"], + } + + try: + r = requests.post( + url=f"{OLLAMA_BASE_URL}/v1/chat/completions", + json=payload, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/integrations/dify_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/dify_pipeline.py new file mode 100644 index 0000000..86e412a --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/dify_pipeline.py @@ -0,0 +1,84 @@ +from typing import List, Union, Generator, Iterator, Optional +from pprint import pprint +import requests, json, warnings + +# Uncomment to disable SSL verification warnings if needed. +# warnings.filterwarnings('ignore', message='Unverified HTTPS request') + +class Pipeline: + def __init__(self): + self.name = "Dify Agent Pipeline" + self.api_url = "http://dify.hostname/v1/workflows/run" # Set correct hostname + self.api_key = "app-dify-key" # Insert your actual API key here.v + self.api_request_stream = True # Dify support stream + self.verify_ssl = True + self.debug = False + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup: {__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is shutdown. + print(f"on_shutdown: {__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called before the OpenAI API request is made. You can modify the form data before it is sent to the OpenAI API. + print(f"inlet: {__name__}") + if self.debug: + print(f"inlet: {__name__} - body:") + pprint(body) + print(f"inlet: {__name__} - user:") + pprint(user) + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called after the OpenAI API response is completed. You can modify the messages after they are received from the OpenAI API. + print(f"outlet: {__name__}") + if self.debug: + print(f"outlet: {__name__} - body:") + pprint(body) + print(f"outlet: {__name__} - user:") + pprint(user) + return body + + def pipe(self, user_message: str, model_id: str, messages: List[dict], body: dict) -> Union[str, Generator, Iterator]: + print(f"pipe: {__name__}") + + if self.debug: + print(f"pipe: {__name__} - received message from user: {user_message}") + + # Set reponse mode Dify API parameter + if self.api_request_stream is True: + response_mode = "streaming" + else: + response_mode = "blocking" + + # This function triggers the workflow using the specified API. + headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Content-Type': 'application/json' + } + data = { + "inputs": {"prompt": user_message}, + "response_mode": response_mode, + "user": body["user"]["email"] + } + + response = requests.post(self.api_url, headers=headers, json=data, stream=self.api_request_stream, verify=self.verify_ssl) + if response.status_code == 200: + # Process and yield each chunk from the response + for line in response.iter_lines(): + if line: + try: + # Remove 'data: ' prefix and parse JSON + json_data = json.loads(line.decode('utf-8').replace('data: ', '')) + # Extract and yield only the 'text' field from the nested 'data' object + if 'data' in json_data and 'text' in json_data['data']: + yield json_data['data']['text'] + except json.JSONDecodeError: + print(f"Failed to parse JSON: {line}") + else: + yield f"Workflow request failed with status code: {response.status_code}" diff --git a/openwebui/pipelines/examples/pipelines/integrations/flowise_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/flowise_pipeline.py new file mode 100644 index 0000000..d57d1bd --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/flowise_pipeline.py @@ -0,0 +1,428 @@ +""" +title: FlowiseAI Integration +author: Eric Zavesky +author_url: https://github.com/ezavesky +git_url: https://github.com/open-webui/pipelines/ +description: Access FlowiseAI endpoints via chat integration +required_open_webui_version: 0.4.3 +requirements: requests,flowise>=1.0.4 +version: 0.4.3 +licence: MIT +""" + +from typing import List, Union, Generator, Iterator, Dict, Optional +from pydantic import BaseModel, Field +import requests +import os +import re +import json +from datetime import datetime +import time +from flowise import Flowise, PredictionData + +from logging import getLogger +logger = getLogger(__name__) +logger.setLevel("DEBUG") + + +class Pipeline: + class Valves(BaseModel): + FLOWISE_API_KEY: str = Field(default="", description="FlowiseAI API key (from Bearer key, e.g. QMknVTFTB40Pk23n6KIVRgdB7va2o-Xlx73zEfpeOu0)") + FLOWISE_BASE_URL: str = Field(default="", description="FlowiseAI base URL (e.g. http://localhost:3000 (URL before '/api/v1/prediction'))") + RATE_LIMIT: int = Field(default=5, description="Rate limit for the pipeline (ops/minute)") + + FLOW_0_ENABLED: Optional[bool] = Field(default=False, description="Flow 0 Enabled (make this flow available for use)") + FLOW_0_ID: Optional[str] = Field(default=None, description="Flow 0 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_0_NAME: Optional[str] = Field(default=None, description="Flow 0 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_1_ENABLED: Optional[bool] = Field(default=False, description="Flow 1 Enabled (make this flow available for use)") + FLOW_1_ID: Optional[str] = Field(default=None, description="Flow 1 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_1_NAME: Optional[str] = Field(default=None, description="Flow 1 Name (human-readable flwo name, no special characters, e.g. news or stock-reader)") + + FLOW_2_ENABLED: Optional[bool] = Field(default=False, description="Flow 2 Enabled (make this flow available for use)") + FLOW_2_ID: Optional[str] = Field(default=None, description="Flow 2 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_2_NAME: Optional[str] = Field(default=None, description="Flow 2 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_3_ENABLED: Optional[bool] = Field(default=False, description="Flow 3 Enabled (make this flow available for use)") + FLOW_3_ID: Optional[str] = Field(default=None, description="Flow 3 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_3_NAME: Optional[str] = Field(default=None, description="Flow 3 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_4_ENABLED: Optional[bool] = Field(default=False, description="Flow 4 Enabled (make this flow available for use)") + FLOW_4_ID: Optional[str] = Field(default=None, description="Flow 4 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_4_NAME: Optional[str] = Field(default=None, description="Flow 4 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_5_ENABLED: Optional[bool] = Field(default=False, description="Flow 5 Enabled (make this flow available for use)") + FLOW_5_ID: Optional[str] = Field(default=None, description="Flow 5 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_5_NAME: Optional[str] = Field(default=None, description="Flow 5 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_6_ENABLED: Optional[bool] = Field(default=False, description="Flow 6 Enabled (make this flow available for use)") + FLOW_6_ID: Optional[str] = Field(default=None, description="Flow 6 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_6_NAME: Optional[str] = Field(default=None, description="Flow 6 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_7_ENABLED: Optional[bool] = Field(default=False, description="Flow 7 Enabled (make this flow available for use)") + FLOW_7_ID: Optional[str] = Field(default=None, description="Flow 7 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_7_NAME: Optional[str] = Field(default=None, description="Flow 7 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_8_ENABLED: Optional[bool] = Field(default=False, description="Flow 8 Enabled (make this flow available for use)") + FLOW_8_ID: Optional[str] = Field(default=None, description="Flow 8 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_8_NAME: Optional[str] = Field(default=None, description="Flow 8 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + FLOW_9_ENABLED: Optional[bool] = Field(default=False, description="Flow 9 Enabled (make this flow available for use)") + FLOW_9_ID: Optional[str] = Field(default=None, description="Flow 9 ID (the flow GUID, e.g. b06d97f5-da14-4d29-81bd-8533261b6c88)") + FLOW_9_NAME: Optional[str] = Field(default=None, description="Flow 9 Name (human-readable flow name, no special characters, e.g. news or stock-reader)") + + + + def __init__(self): + self.name = "FlowiseAI Pipeline" + + # Initialize valve parameters from environment variables + self.valves = self.Valves( + **{k: os.getenv(k, v.default) for k, v in self.Valves.model_fields.items()} + ) + + # Build flow mapping for faster lookup + self.flows = {} + self.update_flows() + + def get_flow_details(self, flow_id: str) -> Optional[dict]: + """ + Fetch flow details from the FlowiseAI API + + Args: + flow_id (str): The ID of the flow to fetch + + Returns: + Optional[dict]: Flow details if successful, None if failed + """ + try: + api_url = f"{self.valves.FLOWISE_BASE_URL.rstrip('/')}/api/v1/chatflows/{flow_id}" + headers = {"Authorization": f"Bearer {self.valves.FLOWISE_API_KEY}"} + + response = requests.get(api_url, headers=headers) + + if response.status_code == 200: + data = response.json() + return data + else: + logger.error(f"Error fetching flow details: Status {response.status_code}") + return None + + except Exception as e: + logger.error(f"Error fetching flow details: {str(e)}") + return None + + def update_flows(self): + """Update the flows dictionary based on the current valve settings""" + self.flows = {} + # Iterate through each flow + for i in range(20): # Support up to 20 flows + enabled_name = f"FLOW_{i}_ENABLED" + if not hasattr(self.valves, enabled_name): # sequential numbering + break + enabled = getattr(self.valves, f"FLOW_{i}_ENABLED", False) + flow_id = getattr(self.valves, f"FLOW_{i}_ID", None) + flow_name = getattr(self.valves, f"FLOW_{i}_NAME", None) + + if enabled and flow_id and flow_name: + # Fetch flow details from API + flow_details = self.get_flow_details(flow_id) + api_name = flow_details.get('name', 'Unknown') if flow_details else 'Unknown' + + # Store both names in the flows dictionary + self.flows[flow_name.lower()] = { + 'id': flow_id, + 'brief_name': flow_name, + 'api_name': api_name + } + + logger.info(f"Updated flows: {[{k: v['api_name']} for k, v in self.flows.items()]}") + + async def on_startup(self): + """Called when the server is started""" + logger.debug(f"on_startup:{self.name}") + self.update_flows() + + async def on_shutdown(self): + """Called when the server is stopped""" + logger.debug(f"on_shutdown:{self.name}") + + async def on_valves_updated(self) -> None: + """Called when valves are updated""" + logger.debug(f"on_valves_updated:{self.name}") + self.update_flows() + + def rate_check(self, dt_start: datetime) -> bool: + """ + Check time, sleep if not enough time has passed for rate + + Args: + dt_start (datetime): Start time of the operation + Returns: + bool: True if sleep was done + """ + dt_end = datetime.now() + time_diff = (dt_end - dt_start).total_seconds() + time_buffer = (1 / self.valves.RATE_LIMIT) + if time_diff >= time_buffer: # no need to sleep + return False + time.sleep(time_buffer - time_diff) + return True + + def parse_user_input(self, user_message: str) -> tuple[str, str]: + """ + Parse the user message to extract flow name and query + + Format expected: @flow_name: query + + Args: + user_message (str): User's input message + + Returns: + tuple[str, str]: Flow name and query + """ + # Match pattern flow_name: query + pattern = r"^([^:]+):\s*(.+)$" + match = re.match(pattern, user_message.strip()) + + if not match: + return None, user_message + + flow_name = match.group(1).strip().lower() + query = match.group(2).strip() + + date_now = datetime.now().strftime("%Y-%m-%d") + time_now = datetime.now().strftime("%H:%M:%S") + query = f"{query}; today's date is {date_now} and the current time is {time_now}" + + return flow_name, query + + def pipe( + self, + user_message: str, + model_id: str, + messages: List[dict], + body: dict + ) -> Union[str, Generator, Iterator]: + """ + Main pipeline function. Calls a specified FlowiseAI flow with the provided query. + + Format expected: @flow_name: query + If no flow is specified, a list of available flows will be returned. + """ + logger.debug(f"pipe:{self.name}") + + dt_start = datetime.now() + streaming = body.get("stream", False) + logger.warning(f"Stream: {streaming}") + context = "" + + # Check if we have valid API configuration + if not self.valves.FLOWISE_API_KEY or not self.valves.FLOWISE_BASE_URL: + error_msg = "FlowiseAI configuration missing. Please set FLOWISE_API_KEY and FLOWISE_BASE_URL valves." + if streaming: + yield error_msg + else: + return error_msg + + # Parse the user message to extract flow name and query + flow_name, query = self.parse_user_input(user_message) + + # If no flow specified or invalid flow, list available flows + if flow_name is None or flow_name not in self.flows: + available_flows = list(self.flows.keys()) + if not available_flows: + no_flows_msg = "No flows configured. Enable at least one FLOW_X_ENABLED valve and set its ID and NAME." + if streaming: + yield no_flows_msg + else: + return no_flows_msg + + flows_list = "\n".join([f"- flow_name: {flow} (description:{self.flows[flow]['api_name']})" for flow in available_flows]) + help_msg = f"Please specify a flow using the format: : \n\nAvailable flows:\n{flows_list}" + + if flow_name is None: + help_msg = "No flow specified. " + help_msg + else: + help_msg = f"Invalid flow '{flow_name}'. " + help_msg + + if streaming: + yield help_msg + return + else: + return help_msg + + # Get the flow ID from the map + flow_id = self.flows[flow_name]['id'] + + if streaming: + yield from self.stream_retrieve(flow_id, flow_name, query, dt_start) + else: + for chunk in self.static_retrieve(flow_id, flow_name, query, dt_start): + context += chunk + return context if context else "No response from FlowiseAI" + + def stream_retrieve( + self, flow_id: str, flow_name: str, query: str, dt_start: datetime + ) -> Generator: + """ + Stream responses from FlowiseAI using the official client library. + + Args: + flow_id (str): The ID of the flow to call + flow_name (str): The name of the flow (for logging) + query (str): The user's query + dt_start (datetime): Start time for rate limiting + + Returns: + Generator: Response chunks for streaming + """ + if not query: + yield "Query is empty. Please provide a question or prompt for the flow." + return + + try: + logger.info(f"Streaming from FlowiseAI flow '{flow_name}' with query: {query}") + + # Rate limiting check + self.rate_check(dt_start) + + # Initialize Flowise client with API configuration + client = Flowise( + base_url=self.valves.FLOWISE_BASE_URL.rstrip('/'), + api_key=self.valves.FLOWISE_API_KEY + ) + + # Create streaming prediction request + completion = client.create_prediction( + PredictionData( + chatflowId=flow_id, + question=query, + streaming=True + ) + ) + + except Exception as e: + error_msg = f"Error streaming from FlowiseAI: {str(e)}" + logger.error(error_msg) + yield error_msg + + idx_last_update = 0 + yield f"Analysis started... {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" + + # Process each streamed chunk + for chunk in completion: + try: + if isinstance(chunk, str): + chunk = json.loads(chunk) + except Exception as e: + # If chunk is not a string, it's already a dictionary + pass + + try: + if isinstance(chunk, dict): + # Expected format: {event: "token", data: "content"} + if "event" in chunk: + if ((chunk["event"] in ["start", "update", "agentReasoning"]) and + ("data" in chunk) and (isinstance(chunk["data"], list))): + for data_update in chunk["data"][idx_last_update:]: + # e.g. {"event":"start","data":[{"agentName":"Perspective Explorer","messages":["... + idx_last_update += 1 + yield "\n---\n" + yield f"\n__Reasoning: {data_update['agentName']} ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')})__\n\n" + for message in data_update["messages"]: + yield message # yield message for each agent update + elif chunk["event"] == "end": + # {"event":"end","data":"[DONE]"} + yield "\n---\n" + yield f"\nAnalysis complete. ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')})\n\n" + elif chunk["event"] == "token": + # do nothing, this is the flat output of the flow (final) + pass + elif "error" in chunk: + error_msg = f"Error from FlowiseAI: {chunk['error']}" + logger.error(error_msg) + yield error_msg + else: + # If chunk format is unexpected, yield as is + yield str(chunk) + except Exception as e: + logger.error(f"Error processing chunk: {str(e)}") + yield f"\nUnusual Response Chunk: ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')})\n{str(e)}\n" + yield f"\n---\n" + yield str(chunk) + + return + + def static_retrieve( + self, flow_id: str, flow_name: str, query: str, dt_start: datetime + ) -> Generator: + """ + Call the FlowiseAI endpoint with the specified flow ID and query using REST API. + + Args: + flow_id (str): The ID of the flow to call + flow_name (str): The name of the flow (for logging) + query (str): The user's query + dt_start (datetime): Start time for rate limiting + + Returns: + Generator: Response chunks for non-streaming requests + """ + if not query: + yield "Query is empty. Please provide a question or prompt for the flow." + return + + api_url = f"{self.valves.FLOWISE_BASE_URL.rstrip('/')}/api/v1/prediction/{flow_id}" + headers = {"Authorization": f"Bearer {self.valves.FLOWISE_API_KEY}"} + + payload = { + "question": query, + } + + try: + logger.info(f"Calling FlowiseAI flow '{flow_name}' with query: {query}") + + # Rate limiting check + self.rate_check(dt_start) + + response = requests.post(api_url, headers=headers, json=payload) + + if response.status_code != 200: + error_msg = f"Error from FlowiseAI: Status {response.status_code}" + logger.error(f"{error_msg} - {response.text}") + yield error_msg + return + + try: + result = response.json() + + # Format might vary based on flow configuration + # Try common response formats + if isinstance(result, dict): + if "text" in result: + yield result["text"] + elif "answer" in result: + yield result["answer"] + elif "response" in result: + yield result["response"] + elif "result" in result: + yield result["result"] + else: + # If no standard field found, return full JSON as string + yield f"```json\n{json.dumps(result, indent=2)}\n```" + elif isinstance(result, str): + yield result + else: + yield f"```json\n{json.dumps(result, indent=2)}\n```" + + except json.JSONDecodeError: + # If not JSON, return the raw text + yield response.text + + except Exception as e: + error_msg = f"Error calling FlowiseAI: {str(e)}" + logger.error(error_msg) + yield error_msg + + return \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/README.md b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/README.md new file mode 100644 index 0000000..8d2cca6 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/README.md @@ -0,0 +1,28 @@ +# Example of langgraph integration +## Python version: 3.11 +## Feature +1. Using langgraph stream writer and custom mode of stream to integrate langgraph with open webui pipeline. +2. Support \ block display. +## Prerequirement +Install the open webui pipeline. +You can follow the docs : https://docs.openwebui.com/pipelines/#-quick-start-with-docker + +## Usage +### 1. Upload pipeline file +Upload `langgraph_stream_pipeline.py` to the open webui pipeline. + +### 2. Enable the uploaded pipeline +Properly set up your langgraph api url. + +And choose **"LangGraph stream"** as your model. + +### 2. Install dependencies +Under the folder `pipelines/examples/pipelines/integrations/langgraph_pipeline`, run command below : +``` +pip install -r requirements.txt +``` +### 3. Start langgraph api server +Run command below : +``` +uvicorn langgraph_example:app --reload +``` \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_example.py b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_example.py new file mode 100644 index 0000000..6ae57a2 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_example.py @@ -0,0 +1,166 @@ +""" +title: Langgraph stream integration +author: bartonzzx +author_url: https://github.com/bartonzzx +git_url: +description: Integrate langgraph with open webui pipeline +required_open_webui_version: 0.4.3 +requirements: none +version: 0.4.3 +licence: MIT +""" + + +import os +import json +import getpass +from typing import Annotated, Literal +from typing_extensions import TypedDict + +from fastapi import FastAPI +from fastapi.responses import StreamingResponse + +from langgraph.graph import StateGraph, START, END +from langgraph.graph.message import add_messages +from langchain_openai import ChatOpenAI +from langgraph.config import get_stream_writer + + +''' +Define LLM API key +''' +def _set_env(var: str): + if not os.environ.get(var): + os.environ[var] = getpass.getpass(f"{var}: ") + + +_set_env("OPENAI_API_KEY") + + +''' +Define Langgraph +''' +def generate_custom_stream(type: Literal["think","normal"], content: str): + content = "\n"+content+"\n" + custom_stream_writer = get_stream_writer() + return custom_stream_writer({type:content}) + +class State(TypedDict): + messages: Annotated[list, add_messages] + +llm = ChatOpenAI(model="gpt-3.5-turbo") + +def chatbot(state: State): + think_response = llm.invoke(["Please reasoning:"] + state["messages"]) + normal_response = llm.invoke(state["messages"]) + generate_custom_stream("think", think_response.content) + generate_custom_stream("normal", normal_response.content) + return {"messages": [normal_response]} + +# Define graph +graph_builder = StateGraph(State) + +# Define nodes +graph_builder.add_node("chatbot", chatbot) +graph_builder.add_edge("chatbot", END) + +# Define edges +graph_builder.add_edge(START, "chatbot") + +# Compile graph +graph = graph_builder.compile() + + +''' +Define api processing +''' +app = FastAPI( + title="Langgraph API", + description="Langgraph API", + ) + +@app.get("/test") +async def test(): + return {"message": "Hello World"} + + +@app.post("/stream") +async def stream(inputs: State): + async def event_stream(): + try: + stream_start_msg = { + 'choices': + [ + { + 'delta': {}, + 'finish_reason': None + } + ] + } + + # Stream start + yield f"data: {json.dumps(stream_start_msg)}\n\n" + + # Processing langgraph stream response with block support + async for event in graph.astream(input=inputs, stream_mode="custom"): + print(event) + think_content = event.get("think", None) + normal_content = event.get("normal", None) + + think_msg = { + 'choices': + [ + { + 'delta': + { + 'reasoning_content': think_content, + }, + 'finish_reason': None + } + ] + } + + normal_msg = { + 'choices': + [ + { + 'delta': + { + 'content': normal_content, + }, + 'finish_reason': None + } + ] + } + + yield f"data: {json.dumps(think_msg)}\n\n" + yield f"data: {json.dumps(normal_msg)}\n\n" + + # End of the stream + stream_end_msg = { + 'choices': [ + { + 'delta': {}, + 'finish_reason': 'stop' + } + ] + } + yield f"data: {json.dumps(stream_end_msg)}\n\n" + + except Exception as e: + # Simply print the error information + print(f"An error occurred: {e}") + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + } + ) + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=9000) \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_stream_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_stream_pipeline.py new file mode 100644 index 0000000..65da0df --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/langgraph_stream_pipeline.py @@ -0,0 +1,63 @@ +""" +title: Langgraph stream integration +author: bartonzzx +author_url: https://github.com/bartonzzx +git_url: +description: Integrate langgraph with open webui pipeline +required_open_webui_version: 0.4.3 +requirements: none +version: 0.4.3 +licence: MIT +""" + + +import os +import requests +from pydantic import BaseModel, Field +from typing import List, Union, Generator, Iterator + + +class Pipeline: + class Valves(BaseModel): + API_URL: str = Field(default="http://127.0.0.1:9000/stream", description="Langgraph API URL") + + def __init__(self): + self.id = "LangGraph stream" + self.name = "LangGraph stream" + # Initialize valve paramaters + self.valves = self.Valves( + **{k: os.getenv(k, v.default) for k, v in self.Valves.model_fields.items()} + ) + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup: {__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is shutdown. + print(f"on_shutdown: {__name__}") + pass + + def pipe( + self, + user_message: str, + model_id: str, + messages: List[dict], + body: dict + ) -> Union[str, Generator, Iterator]: + + data = { + "messages": [[msg['role'], msg['content']] for msg in messages], + } + + headers = { + 'accept': 'text/event-stream', + 'Content-Type': 'application/json', + } + + response = requests.post(self.valves.API_URL, json=data, headers=headers, stream=True) + + response.raise_for_status() + + return response.iter_lines() \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/requirements.txt b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/requirements.txt new file mode 100644 index 0000000..fc122d6 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/langgraph_pipeline/requirements.txt @@ -0,0 +1,40 @@ +annotated-types==0.7.0 +anyio==4.8.0 +certifi==2025.1.31 +charset-normalizer==3.4.1 +click==8.1.8 +distro==1.9.0 +fastapi==0.115.11 +h11==0.14.0 +httpcore==1.0.7 +httpx==0.28.1 +idna==3.10 +jiter==0.9.0 +jsonpatch==1.33 +jsonpointer==3.0.0 +langchain-core==0.3.45 +langchain-openai==0.3.8 +langgraph==0.3.11 +langgraph-checkpoint==2.0.20 +langgraph-prebuilt==0.1.3 +langgraph-sdk==0.1.57 +langsmith==0.3.15 +msgpack==1.1.0 +openai==1.66.3 +orjson==3.10.15 +packaging==24.2 +pydantic==2.10.6 +pydantic_core==2.27.2 +PyYAML==6.0.2 +regex==2024.11.6 +requests==2.32.3 +requests-toolbelt==1.0.0 +sniffio==1.3.1 +starlette==0.46.1 +tenacity==9.0.0 +tiktoken==0.9.0 +tqdm==4.67.1 +typing_extensions==4.12.2 +urllib3==2.3.0 +uvicorn==0.34.0 +zstandard==0.23.0 diff --git a/openwebui/pipelines/examples/pipelines/integrations/n8n_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/n8n_pipeline.py new file mode 100644 index 0000000..51e0e4d --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/n8n_pipeline.py @@ -0,0 +1,79 @@ +from typing import List, Union, Generator, Iterator, Optional +from pprint import pprint +import requests, json, warnings + +# Uncomment to disable SSL verification warnings if needed. +# warnings.filterwarnings('ignore', message='Unverified HTTPS request') + +class Pipeline: + def __init__(self): + self.name = "N8N Agent Pipeline" + self.api_url = "https://n8n.host/webhook/myflow" # Set correct hostname + self.api_key = "" # Insert your actual API key here + self.verify_ssl = True + self.debug = False + # Please note that N8N do not support stream reponses + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup: {__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is shutdown. + print(f"on_shutdown: {__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called before the OpenAI API request is made. You can modify the form data before it is sent to the OpenAI API. + print(f"inlet: {__name__}") + if self.debug: + print(f"inlet: {__name__} - body:") + pprint(body) + print(f"inlet: {__name__} - user:") + pprint(user) + return body + + async def outlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This function is called after the OpenAI API response is completed. You can modify the messages after they are received from the OpenAI API. + print(f"outlet: {__name__}") + if self.debug: + print(f"outlet: {__name__} - body:") + pprint(body) + print(f"outlet: {__name__} - user:") + pprint(user) + return body + + def pipe(self, user_message: str, model_id: str, messages: List[dict], body: dict) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe: {__name__}") + + if self.debug: + print(f"pipe: {__name__} - received message from user: {user_message}") + + # This function triggers the workflow using the specified API. + headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Content-Type': 'application/json' + } + data = { + "inputs": {"prompt": user_message}, + "user": body["user"]["email"] + } + + response = requests.post(self.api_url, headers=headers, json=data, verify=self.verify_ssl) + if response.status_code == 200: + # Process and yield each chunk from the response + try: + for line in response.iter_lines(): + if line: + # Decode each line assuming UTF-8 encoding and directly parse it as JSON + json_data = json.loads(line.decode('utf-8')) + # Check if 'output' exists in json_data and yield it + if 'output' in json_data: + yield json_data['output'] + except json.JSONDecodeError as e: + print(f"Failed to parse JSON from line. Error: {str(e)}") + yield "Error in JSON parsing." + else: + yield f"Workflow request failed with status code: {response.status_code}" diff --git a/openwebui/pipelines/examples/pipelines/integrations/python_code_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/python_code_pipeline.py new file mode 100644 index 0000000..938d984 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/python_code_pipeline.py @@ -0,0 +1,50 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import subprocess + + +class Pipeline: + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "python_code_pipeline" + self.name = "Python Code Pipeline" + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def execute_python_code(self, code): + try: + result = subprocess.run( + ["python", "-c", code], capture_output=True, text=True, check=True + ) + stdout = result.stdout.strip() + return stdout, result.returncode + except subprocess.CalledProcessError as e: + return e.output.strip(), e.returncode + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + if body.get("title", False): + print("Title Generation") + return "Python Code Pipeline" + else: + stdout, return_code = self.execute_python_code(user_message) + return stdout diff --git a/openwebui/pipelines/examples/pipelines/integrations/wikipedia_pipeline.py b/openwebui/pipelines/examples/pipelines/integrations/wikipedia_pipeline.py new file mode 100644 index 0000000..15eb797 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/integrations/wikipedia_pipeline.py @@ -0,0 +1,218 @@ +""" +title: Wikipedia Article Retrieval +author: Unknown +author_url: Unknown +git_url: https://github.com/open-webui/pipelines/blob/main/examples/pipelines/integrations/wikipedia_pipeline.py +description: Wikipedia Search and Return +required_open_webui_version: 0.4.3 +requirements: wikipedia +version: 0.4.3 +licence: MIT +""" + + +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel, Field +import wikipedia +import requests +import os +from datetime import datetime +import time +import re + +from logging import getLogger +logger = getLogger(__name__) +logger.setLevel("DEBUG") + + +class Pipeline: + class Valves(BaseModel): + # OPENAI_API_KEY: str = Field(default="", description="OpenAI API key") + RATE_LIMIT: int = Field(default=5, description="Rate limit for the pipeline") + WORD_LIMIT: int = Field(default=300, description="Word limit when getting page summary") + WIKIPEDIA_ROOT: str = Field(default="https://en.wikipedia.org/wiki", description="Wikipedia root URL") + + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "wiki_pipeline" + self.name = "Wikipedia Pipeline" + + # Initialize valve paramaters + self.valves = self.Valves( + **{k: os.getenv(k, v.default) for k, v in self.Valves.model_fields.items()} + ) + + async def on_startup(self): + # This function is called when the server is started. + logger.debug(f"on_startup:{self.name}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + logger.debug(f"on_shutdown:{self.name}") + pass + + def rate_check(self, dt_start: datetime): + """ + Check time, sleep if not enough time has passed for rate + + Args: + dt_start (datetime): Start time of the operation + Returns: + bool: True if sleep was done + """ + dt_end = datetime.now() + time_diff = (dt_end - dt_start).total_seconds() + time_buffer = (1 / self.valves.RATE_LIMIT) + if time_diff >= time_buffer: # no need to sleep + return False + time.sleep(time_buffer - time_diff) + return True + + def pipe( + self, + user_message: str, + model_id: str, + messages: List[dict], + body: dict + ) -> Union[str, Generator, Iterator]: + """ + Main pipeline function. Performs wikipedia article lookup by query + and returns the summary of the first article. + """ + logger.debug(f"pipe:{self.name}") + + # Check if title generation is requested + # as of 12/28/24, these were standard greetings + if ("broad tags categorizing" in user_message.lower()) \ + or ("Create a concise" in user_message.lower()): + # ## Create a concise, 3-5 word title with + # ## Task:\nGenerate 1-3 broad tags categorizing the main themes + logger.debug(f"Title Generation (aborted): {user_message}") + return "(title generation disabled)" + + logger.info(f"User Message: {user_message}") + # logger.info(f"Messages: {messages}") + # [{'role': 'user', 'content': 'history of ibm'}] + + # logger.info(f"Body: {body}") + # {'stream': True, 'model': 'wikipedia_pipeline', + # 'messages': [{'role': 'user', 'content': 'history of ibm'}], + # 'user': {'name': 'User', 'id': '235a828f-84a3-44a0-b7af-721ee8be6571', + # 'email': 'admin@localhost', 'role': 'admin'}} + + dt_start = datetime.now() + multi_part = False + streaming = body.get("stream", False) + logger.warning(f"Stream: {streaming}") + context = "" + + # examples from https://pypi.org/project/wikipedia/ + # new addition - ability to include multiple topics with a semicolon + for query in user_message.split(';'): + self.rate_check(dt_start) + query = query.strip() + + if multi_part: + if streaming: + yield "---\n" + else: + context += "---\n" + if body.get("stream", True): + yield from self.stream_retrieve(query, dt_start) + else: + for chunk in self.stream_retrieve(query, dt_start): + context += chunk + multi_part = True + + if not streaming: + return context if context else "No information found" + + + def stream_retrieve( + self, query:str, dt_start: datetime, + ) -> Generator: + """ + Retrieve the wikipedia page for the query and return the summary. Return a generator + for streaming responses but can also be iterated for a single response. + """ + + re_query = re.compile(r"[^0-9A-Z]", re.IGNORECASE) + re_rough_word = re.compile(r"[\w]+", re.IGNORECASE) + + titles_found = None + try: + titles_found = wikipedia.search(query) + # r = requests.get( + # f"https://en.wikipedia.org/w/api.php?action=opensearch&search={query}&limit=1&namespace=0&format=json" + # ) + logger.info(f"Query: {query}, Found: {titles_found}") + except Exception as e: + logger.error(f"Search Error: {query} -> {e}") + yield f"Page Search Error: {query}" + + if titles_found is None or not titles_found: # no results + yield f"No information found for '{query}'" + return + + self.rate_check(dt_start) + + # if context: # add separator if multiple topics + # context += "---\n" + try: + title_check = titles_found[0] + wiki_page = wikipedia.page(title_check, auto_suggest=False) # trick! don't auto-suggest + except wikipedia.exceptions.DisambiguationError as e: + str_error = str(e).replace("\n", ", ") + str_error = f"## Disambiguation Error ({query})\n* Status: {str_error}" + logger.error(str_error) + yield str_error + "\n" + return + except wikipedia.exceptions.RedirectError as e: + str_error = str(e).replace("\n", ", ") + str_error = f"## Redirect Error ({query})\n* Status: {str_error}" + logger.error(str_error) + yield str_error + "\n" + return + except Exception as e: + if titles_found: + str_error = f"## Page Retrieve Error ({query})\n* Found Topics (matched '{title_check}') {titles_found}" + logger.error(f"{str_error} -> {e}") + else: + str_error = f"## Page Not Found ({query})\n* Unknown error" + logger.error(f"{str_error} -> {e}") + yield str_error + "\n" + return + + # found a page / section + logger.info(f"Page Sections[{query}]: {wiki_page.sections}") + yield f"## {title_check}\n" + + # flatten internal links + # link_md = [f"[{x}]({self.valves.WIKIPEDIA_ROOT}/{re_query.sub('_', x)})" for x in wiki_page.links[:10]] + # yield "* Links (first 30): " + ",".join(link_md) + "\n" + + # add the textual summary + summary_full = wiki_page.summary + word_positions = [x.start() for x in re_rough_word.finditer(summary_full)] + if len(word_positions) > self.valves.WORD_LIMIT: + yield summary_full[:word_positions[self.valves.WORD_LIMIT]] + "...\n" + else: + yield summary_full + "\n" + + # the more you know! link to further reading + yield "### Learn More" + "\n" + yield f"* [Read more on Wikipedia...]({wiki_page.url})\n" + + # also spit out the related topics from search + link_md = [f"[{x}]({self.valves.WIKIPEDIA_ROOT}/{re_query.sub('_', x)})" for x in titles_found] + yield f"* Related topics: {', '.join(link_md)}\n" + + # throw in the first image for good measure + if wiki_page.images: + yield f"\n![Image: {title_check}]({wiki_page.images[0]})\n" + + return \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/providers/anthropic_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/anthropic_manifold_pipeline.py new file mode 100644 index 0000000..f8a4c67 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/anthropic_manifold_pipeline.py @@ -0,0 +1,293 @@ +""" +title: Anthropic Manifold Pipeline +author: justinh-rahb, sriparashiva +date: 2024-06-20 +version: 1.4 +license: MIT +description: A pipeline for generating text and processing images using the Anthropic API. +requirements: requests, sseclient-py +environment_variables: ANTHROPIC_API_KEY, ANTHROPIC_THINKING_BUDGET_TOKENS, ANTHROPIC_ENABLE_THINKING +""" + +import os +import requests +import json +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import sseclient + +from utils.pipelines.main import pop_system_message + +REASONING_EFFORT_BUDGET_TOKEN_MAP = { + "none": None, + "low": 1024, + "medium": 4096, + "high": 16384, + "max": 32768, +} + +# Maximum combined token limit for Claude 3.7 +MAX_COMBINED_TOKENS = 64000 + + +class Pipeline: + class Valves(BaseModel): + ANTHROPIC_API_KEY: str = "" + + def __init__(self): + self.type = "manifold" + self.id = "anthropic" + self.name = "anthropic/" + + self.valves = self.Valves( + **{ + "ANTHROPIC_API_KEY": os.getenv( + "ANTHROPIC_API_KEY", "your-api-key-here" + ), + } + ) + self.url = "https://api.anthropic.com/v1/messages" + self.update_headers() + + def update_headers(self): + self.headers = { + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "x-api-key": self.valves.ANTHROPIC_API_KEY, + } + + def get_anthropic_models(self): + return [ + {"id": "claude-3-haiku-20240307", "name": "claude-3-haiku"}, + {"id": "claude-3-opus-20240229", "name": "claude-3-opus"}, + {"id": "claude-3-sonnet-20240229", "name": "claude-3-sonnet"}, + {"id": "claude-3-5-haiku-20241022", "name": "claude-3.5-haiku"}, + {"id": "claude-3-5-sonnet-20241022", "name": "claude-3.5-sonnet"}, + {"id": "claude-3-7-sonnet-20250219", "name": "claude-3.7-sonnet"}, + {"id": "claude-opus-4-20250514", "name": "claude-4-opus"}, + {"id": "claude-sonnet-4-20250514", "name": "claude-4-sonnet"}, + {"id": "claude-opus-4-1-20250805", "name": "claude-4.1-opus"}, + ] + + def get_thinking_supported_models(self): + """Returns list of model identifiers that support extended thinking""" + return [ + "claude-3-7", + "claude-sonnet-4", + "claude-opus-4" + ] + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + self.update_headers() + + def pipelines(self) -> List[dict]: + return self.get_anthropic_models() + + def process_image(self, image_data): + if image_data["url"].startswith("data:image"): + mime_type, base64_data = image_data["url"].split(",", 1) + media_type = mime_type.split(":")[1].split(";")[0] + return { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": base64_data, + }, + } + else: + return { + "type": "image", + "source": {"type": "url", "url": image_data["url"]}, + } + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + try: + # Remove unnecessary keys + for key in ["user", "chat_id", "title"]: + body.pop(key, None) + + system_message, messages = pop_system_message(messages) + + processed_messages = [] + image_count = 0 + total_image_size = 0 + + for message in messages: + processed_content = [] + if isinstance(message.get("content"), list): + for item in message["content"]: + if item["type"] == "text": + processed_content.append( + {"type": "text", "text": item["text"]} + ) + elif item["type"] == "image_url": + if image_count >= 5: + raise ValueError( + "Maximum of 5 images per API call exceeded" + ) + + processed_image = self.process_image(item["image_url"]) + processed_content.append(processed_image) + + if processed_image["source"]["type"] == "base64": + image_size = ( + len(processed_image["source"]["data"]) * 3 / 4 + ) + else: + image_size = 0 + + total_image_size += image_size + if total_image_size > 100 * 1024 * 1024: + raise ValueError( + "Total size of images exceeds 100 MB limit" + ) + + image_count += 1 + else: + processed_content = [ + {"type": "text", "text": message.get("content", "")} + ] + + processed_messages.append( + {"role": message["role"], "content": processed_content} + ) + + # Prepare the payload + payload = { + "model": model_id, + "messages": processed_messages, + "max_tokens": body.get("max_tokens", 4096), + "temperature": body.get("temperature", 0.8), + "stop_sequences": body.get("stop", []), + **({"system": str(system_message)} if system_message else {}), + "stream": body.get("stream", False), + } + + # Add optional parameters only if explicitly provided + if "top_k" in body: + payload["top_k"] = body["top_k"] + + # Only include top_p if explicitly set (not both temperature and top_p) + if "top_p" in body: + payload["top_p"] = body["top_p"] + # Remove temperature if top_p is explicitly set + if "temperature" in payload: + del payload["temperature"] + + if body.get("stream", False): + supports_thinking = any(model in model_id for model in self.get_thinking_supported_models()) + reasoning_effort = body.get("reasoning_effort", "none") + budget_tokens = REASONING_EFFORT_BUDGET_TOKEN_MAP.get(reasoning_effort) + + # Allow users to input an integer value representing budget tokens + if ( + not budget_tokens + and reasoning_effort is not None + and reasoning_effort not in REASONING_EFFORT_BUDGET_TOKEN_MAP.keys() + ): + try: + budget_tokens = int(reasoning_effort) + except ValueError as e: + print("Failed to convert reasoning effort to int", e) + budget_tokens = None + + if supports_thinking and budget_tokens: + # Check if the combined tokens (budget_tokens + max_tokens) exceeds the limit + max_tokens = payload.get("max_tokens", 4096) + combined_tokens = budget_tokens + max_tokens + + if combined_tokens > MAX_COMBINED_TOKENS: + error_message = f"Error: Combined tokens (budget_tokens {budget_tokens} + max_tokens {max_tokens} = {combined_tokens}) exceeds the maximum limit of {MAX_COMBINED_TOKENS}" + print(error_message) + return error_message + + payload["max_tokens"] = combined_tokens + payload["thinking"] = { + "type": "enabled", + "budget_tokens": budget_tokens, + } + # Thinking requires temperature 1.0 and does not support top_p, top_k + payload["temperature"] = 1.0 + if "top_k" in payload: + del payload["top_k"] + if "top_p" in payload: + del payload["top_p"] + return self.stream_response(payload) + else: + return self.get_completion(payload) + except Exception as e: + return f"Error: {e}" + + def stream_response(self, payload: dict) -> Generator: + """Used for title and tag generation""" + try: + response = requests.post( + self.url, headers=self.headers, json=payload, stream=True + ) + print(f"{response} for {payload}") + + if response.status_code == 200: + client = sseclient.SSEClient(response) + for event in client.events(): + try: + data = json.loads(event.data) + if data["type"] == "content_block_start": + if data["content_block"]["type"] == "thinking": + yield "" + else: + yield data["content_block"]["text"] + elif data["type"] == "content_block_delta": + if data["delta"]["type"] == "thinking_delta": + yield data["delta"]["thinking"] + elif data["delta"]["type"] == "signature_delta": + yield "\n \n\n" + else: + yield data["delta"]["text"] + elif data["type"] == "message_stop": + break + except json.JSONDecodeError: + print(f"Failed to parse JSON: {event.data}") + yield f"Error: Failed to parse JSON response" + except KeyError as e: + print(f"Unexpected data structure: {e} for payload {payload}") + print(f"Full data: {data}") + yield f"Error: Unexpected data structure: {e}" + else: + error_message = f"Error: {response.status_code} - {response.text}" + print(error_message) + yield error_message + except Exception as e: + error_message = f"Error: {str(e)}" + print(error_message) + yield error_message + + def get_completion(self, payload: dict) -> str: + try: + response = requests.post(self.url, headers=self.headers, json=payload) + print(response, payload) + if response.status_code == 200: + res = response.json() + for content in res["content"]: + if not content.get("text"): + continue + return content["text"] + return "" + else: + error_message = f"Error: {response.status_code} - {response.text}" + print(error_message) + return error_message + except Exception as e: + error_message = f"Error: {str(e)}" + print(error_message) + return error_message diff --git a/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_claude_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_claude_pipeline.py new file mode 100644 index 0000000..1a5a028 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_claude_pipeline.py @@ -0,0 +1,285 @@ +""" +title: AWS Bedrock Claude Pipeline +author: G-mario +date: 2024-08-18 +version: 1.0 +license: MIT +description: A pipeline for generating text and processing images using the AWS Bedrock API(By Anthropic claude). +requirements: requests, boto3 +environment_variables: AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION_NAME +""" +import base64 +import json +import logging +from io import BytesIO +from typing import List, Union, Generator, Iterator, Optional, Any + +import boto3 + +from pydantic import BaseModel + +import os +import requests + +from utils.pipelines.main import pop_system_message + +REASONING_EFFORT_BUDGET_TOKEN_MAP = { + "none": None, + "low": 1024, + "medium": 4096, + "high": 16384, + "max": 32768, +} + +# Maximum combined token limit for Claude 3.7 +MAX_COMBINED_TOKENS = 64000 + + +class Pipeline: + class Valves(BaseModel): + AWS_ACCESS_KEY: Optional[str] = None + AWS_SECRET_KEY: Optional[str] = None + AWS_REGION_NAME: Optional[str] = None + + def __init__(self): + self.type = "manifold" + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "openai_pipeline" + self.name = "Bedrock: " + + self.valves = self.Valves( + **{ + "AWS_ACCESS_KEY": os.getenv("AWS_ACCESS_KEY", ""), + "AWS_SECRET_KEY": os.getenv("AWS_SECRET_KEY", ""), + "AWS_REGION_NAME": os.getenv( + "AWS_REGION_NAME", os.getenv( + "AWS_REGION", os.getenv("AWS_DEFAULT_REGION", "") + ) + ), + } + ) + + self.update_pipelines() + + def get_thinking_supported_models(self): + """Returns list of model identifiers that support extended thinking""" + return [ + "claude-3-7", + "claude-sonnet-4", + "claude-opus-4" + ] + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + self.update_pipelines() + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + self.update_pipelines() + + def update_pipelines(self) -> None: + try: + self.bedrock = boto3.client(service_name="bedrock", + aws_access_key_id=self.valves.AWS_ACCESS_KEY, + aws_secret_access_key=self.valves.AWS_SECRET_KEY, + region_name=self.valves.AWS_REGION_NAME) + self.bedrock_runtime = boto3.client(service_name="bedrock-runtime", + aws_access_key_id=self.valves.AWS_ACCESS_KEY, + aws_secret_access_key=self.valves.AWS_SECRET_KEY, + region_name=self.valves.AWS_REGION_NAME) + self.pipelines = self.get_models() + except Exception as e: + print(f"Error: {e}") + self.pipelines = [ + { + "id": "error", + "name": "Could not fetch models from Bedrock, please set up AWS Key/Secret or Instance/Task Role.", + }, + ] + + def get_models(self): + try: + res = [] + response = self.bedrock.list_foundation_models(byProvider='Anthropic') + for model in response['modelSummaries']: + inference_types = model.get('inferenceTypesSupported', []) + if "ON_DEMAND" in inference_types: + res.append({'id': model['modelId'], 'name': model['modelName']}) + elif "INFERENCE_PROFILE" in inference_types: + inferenceProfileId = self.getInferenceProfileId(model['modelArn']) + if inferenceProfileId: + res.append({'id': inferenceProfileId, 'name': model['modelName']}) + + return res + except Exception as e: + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from Bedrock, please check permissoin.", + }, + ] + + def getInferenceProfileId(self, modelArn: str) -> str: + response = self.bedrock.list_inference_profiles() + for profile in response.get('inferenceProfileSummaries', []): + for model in profile.get('models', []): + if model.get('modelArn') == modelArn: + return profile['inferenceProfileId'] + return None + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + system_message, messages = pop_system_message(messages) + + logging.info(f"pop_system_message: {json.dumps(messages)}") + + try: + processed_messages = [] + image_count = 0 + for message in messages: + processed_content = [] + if isinstance(message.get("content"), list): + for item in message["content"]: + if item["type"] == "text": + processed_content.append({"text": item["text"]}) + elif item["type"] == "image_url": + if image_count >= 20: + raise ValueError("Maximum of 20 images per API call exceeded") + processed_image = self.process_image(item["image_url"]) + processed_content.append(processed_image) + image_count += 1 + else: + processed_content = [{"text": message.get("content", "")}] + + processed_messages.append({"role": message["role"], "content": processed_content}) + + payload = { + "modelId": model_id, + "messages": processed_messages, + "system": [{'text': system_message["content"] if system_message else 'you are an intelligent ai assistant'}], + "inferenceConfig": { + "temperature": body.get("temperature", 0.5), + "maxTokens": body.get("max_tokens", 4096), + "stopSequences": body.get("stop", []), + }, + "additionalModelRequestFields": {} + } + + # Handle top_p and temperature conflict + if "top_p" in body: + payload["inferenceConfig"]["topP"] = body["top_p"] + # Remove temperature if top_p is explicitly set + if "temperature" in payload["inferenceConfig"]: + del payload["inferenceConfig"]["temperature"] + + # Add top_k if explicitly provided + if "top_k" in body: + payload["additionalModelRequestFields"]["top_k"] = body["top_k"] + else: + # Use default top_k value + payload["additionalModelRequestFields"]["top_k"] = 200 + + if body.get("stream", False): + supports_thinking = any(model in model_id for model in self.get_thinking_supported_models()) + reasoning_effort = body.get("reasoning_effort", "none") + budget_tokens = REASONING_EFFORT_BUDGET_TOKEN_MAP.get(reasoning_effort) + + # Allow users to input an integer value representing budget tokens + if ( + not budget_tokens + and reasoning_effort is not None + and reasoning_effort not in REASONING_EFFORT_BUDGET_TOKEN_MAP.keys() + ): + try: + budget_tokens = int(reasoning_effort) + except ValueError as e: + print("Failed to convert reasoning effort to int", e) + budget_tokens = None + + if supports_thinking and budget_tokens: + # Check if the combined tokens (budget_tokens + max_tokens) exceeds the limit + max_tokens = payload.get("max_tokens", 4096) + combined_tokens = budget_tokens + max_tokens + + if combined_tokens > MAX_COMBINED_TOKENS: + error_message = f"Error: Combined tokens (budget_tokens {budget_tokens} + max_tokens {max_tokens} = {combined_tokens}) exceeds the maximum limit of {MAX_COMBINED_TOKENS}" + print(error_message) + return error_message + + payload["inferenceConfig"]["maxTokens"] = combined_tokens + payload["additionalModelRequestFields"]["thinking"] = { + "type": "enabled", + "budget_tokens": budget_tokens, + } + # Thinking requires temperature 1.0 and does not support top_p, top_k + payload["inferenceConfig"]["temperature"] = 1.0 + if "top_k" in payload["additionalModelRequestFields"]: + del payload["additionalModelRequestFields"]["top_k"] + if "topP" in payload["inferenceConfig"]: + del payload["inferenceConfig"]["topP"] + return self.stream_response(model_id, payload) + else: + return self.get_completion(model_id, payload) + except Exception as e: + return f"Error: {e}" + + def process_image(self, image: str): + img_stream = None + content_type = None + + if image["url"].startswith("data:image"): + mime_type, base64_string = image["url"].split(",", 1) + content_type = mime_type.split(":")[1].split(";")[0] + image_data = base64.b64decode(base64_string) + img_stream = BytesIO(image_data) + else: + response = requests.get(image["url"]) + img_stream = BytesIO(response.content) + content_type = response.headers.get('Content-Type', 'image/jpeg') + + media_type = content_type.split('/')[-1] if '/' in content_type else content_type + return { + "image": { + "format": media_type, + "source": {"bytes": img_stream.read()} + } + } + + def stream_response(self, model_id: str, payload: dict) -> Generator: + streaming_response = self.bedrock_runtime.converse_stream(**payload) + + in_resasoning_context = False + for chunk in streaming_response["stream"]: + if in_resasoning_context and "contentBlockStop" in chunk: + in_resasoning_context = False + yield "\n \n\n" + elif "contentBlockDelta" in chunk and "delta" in chunk["contentBlockDelta"]: + if "reasoningContent" in chunk["contentBlockDelta"]["delta"]: + if not in_resasoning_context: + yield "" + + in_resasoning_context = True + if "text" in chunk["contentBlockDelta"]["delta"]["reasoningContent"]: + yield chunk["contentBlockDelta"]["delta"]["reasoningContent"]["text"] + elif "text" in chunk["contentBlockDelta"]["delta"]: + yield chunk["contentBlockDelta"]["delta"]["text"] + + def get_completion(self, model_id: str, payload: dict) -> str: + response = self.bedrock_runtime.converse(**payload) + return response['output']['message']['content'][0]['text'] diff --git a/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_deepseek_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_deepseek_pipeline.py new file mode 100644 index 0000000..8f6512e --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/aws_bedrock_deepseek_pipeline.py @@ -0,0 +1,187 @@ +""" +title: AWS Bedrock DeepSeek Pipeline +author: kikumoto +date: 2025-03-17 +version: 1.0 +license: MIT +description: A pipeline for generating text using the AWS Bedrock API. +requirements: boto3 +environment_variables: +""" + +import json +import logging + +from typing import List, Union, Generator, Iterator, Dict, Optional, Any + +import boto3 + +from pydantic import BaseModel + +import os + +from utils.pipelines.main import pop_system_message + +class Pipeline: + class Valves(BaseModel): + AWS_ACCESS_KEY: Optional[str] = None + AWS_SECRET_KEY: Optional[str] = None + AWS_REGION_NAME: Optional[str] = None + + def __init__(self): + self.type = "manifold" + self.name = "Bedrock DeepSeek: " + + self.valves = self.Valves( + **{ + "AWS_ACCESS_KEY": os.getenv("AWS_ACCESS_KEY", ""), + "AWS_SECRET_KEY": os.getenv("AWS_SECRET_KEY", ""), + "AWS_REGION_NAME": os.getenv( + "AWS_REGION_NAME", os.getenv( + "AWS_REGION", os.getenv("AWS_DEFAULT_REGION", "") + ) + ), + } + ) + + self.update_pipelines() + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + self.update_pipelines() + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + self.update_pipelines() + + def update_pipelines(self) -> None: + try: + self.bedrock = boto3.client(service_name="bedrock", + aws_access_key_id=self.valves.AWS_ACCESS_KEY, + aws_secret_access_key=self.valves.AWS_SECRET_KEY, + region_name=self.valves.AWS_REGION_NAME) + self.bedrock_runtime = boto3.client(service_name="bedrock-runtime", + aws_access_key_id=self.valves.AWS_ACCESS_KEY, + aws_secret_access_key=self.valves.AWS_SECRET_KEY, + region_name=self.valves.AWS_REGION_NAME) + self.pipelines = self.get_models() + except Exception as e: + print(f"Error: {e}") + self.pipelines = [ + { + "id": "error", + "name": "Could not fetch models from Bedrock, please set up AWS Key/Secret or Instance/Task Role.", + }, + ] + + def pipelines(self) -> List[dict]: + return self.get_models() + + def get_models(self): + try: + res = [] + response = self.bedrock.list_foundation_models(byProvider='DeepSeek') + for model in response['modelSummaries']: + inference_types = model.get('inferenceTypesSupported', []) + if "ON_DEMAND" in inference_types: + res.append({'id': model['modelId'], 'name': model['modelName']}) + elif "INFERENCE_PROFILE" in inference_types: + inferenceProfileId = self.getInferenceProfileId(model['modelArn']) + if inferenceProfileId: + res.append({'id': inferenceProfileId, 'name': model['modelName']}) + + return res + except Exception as e: + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from Bedrock, please check permissoin.", + }, + ] + + def getInferenceProfileId(self, modelArn: str) -> str: + response = self.bedrock.list_inference_profiles() + for profile in response.get('inferenceProfileSummaries', []): + for model in profile.get('models', []): + if model.get('modelArn') == modelArn: + return profile['inferenceProfileId'] + return None + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + try: + # Remove unnecessary keys + for key in ['user', 'chat_id', 'title']: + body.pop(key, None) + + system_message, messages = pop_system_message(messages) + + logging.info(f"pop_system_message: {json.dumps(messages)}") + + processed_messages = [] + for message in messages: + processed_content = [] + if isinstance(message.get("content"), list): + for item in message["content"]: + # DeepSeek currently doesn't support multi-modal inputs + if item["type"] == "text": + processed_content.append({"text": item["text"]}) + else: + processed_content = [{"text": message.get("content", "")}] + + processed_messages.append({"role": message["role"], "content": processed_content}) + + payload = {"modelId": model_id, + "system": [{'text': system_message["content"] if system_message else 'you are an intelligent ai assistant'}], + "messages": processed_messages, + "inferenceConfig": { + "temperature": body.get("temperature", 0.5), + "topP": body.get("top_p", 0.9), + "maxTokens": body.get("max_tokens", 8192), + "stopSequences": body.get("stop", []), + }, + } + + if body.get("stream", False): + return self.stream_response(model_id, payload) + else: + return self.get_completion(model_id, payload) + + except Exception as e: + return f"Error: {e}" + + def stream_response(self, model_id: str, payload: dict) -> Generator: + streaming_response = self.bedrock_runtime.converse_stream(**payload) + + in_resasoning_context = False + for chunk in streaming_response["stream"]: + if in_resasoning_context and "contentBlockStop" in chunk: + in_resasoning_context = False + yield "\n \n\n" + elif "contentBlockDelta" in chunk and "delta" in chunk["contentBlockDelta"]: + if "reasoningContent" in chunk["contentBlockDelta"]["delta"]: + if not in_resasoning_context: + yield "" + + in_resasoning_context = True + if "text" in chunk["contentBlockDelta"]["delta"]["reasoningContent"]: + yield chunk["contentBlockDelta"]["delta"]["reasoningContent"]["text"] + elif "text" in chunk["contentBlockDelta"]["delta"]: + yield chunk["contentBlockDelta"]["delta"]["text"] + + def get_completion(self, model_id: str, payload: dict) -> str: + response = self.bedrock_runtime.converse(**payload) + return response['output']['message']['content'][0]['text'] \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/providers/azure_dalle_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/azure_dalle_manifold_pipeline.py new file mode 100644 index 0000000..c64a766 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/azure_dalle_manifold_pipeline.py @@ -0,0 +1,89 @@ +""" +title: Azure - Dall-E Manifold Pipeline +author: weisser-dev +date: 2025-03-26 +version: 1.0 +license: MIT +description: A pipeline for generating text and processing images using the Azure API. And including multiple Dall-e models +requirements: requests, os +environment_variables: AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_MODELS, AZURE_OPENAI_MODEL_NAMES, IMAGE_SIZE, NUM_IMAGES +""" +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import requests +import os + +class Pipeline: + class Valves(BaseModel): + AZURE_OPENAI_API_KEY: str + AZURE_OPENAI_ENDPOINT: str + AZURE_OPENAI_API_VERSION: str + AZURE_OPENAI_MODELS: str + AZURE_OPENAI_MODEL_NAMES: str + IMAGE_SIZE: str = "1024x1024" + NUM_IMAGES: int = 1 + + def __init__(self): + self.type = "manifold" + self.name = "Azure DALLΒ·E: " + self.valves = self.Valves( + **{ + "AZURE_OPENAI_API_KEY": os.getenv("AZURE_OPENAI_API_KEY", "your-azure-openai-api-key-here"), + "AZURE_OPENAI_ENDPOINT": os.getenv("AZURE_OPENAI_ENDPOINT", "your-azure-openai-endpoint-here"), + "AZURE_OPENAI_API_VERSION": os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-01"), + "AZURE_OPENAI_MODELS": os.getenv("AZURE_OPENAI_MODELS", "dall-e-2;dall-e-3"), #ensure that the model here is within your enpoint url, sometime the name within the url it is also like Dalle3 + "AZURE_OPENAI_MODEL_NAMES": os.getenv("AZURE_OPENAI_MODEL_NAMES", "DALL-E 2;DALL-E 3"), + } + ) + self.set_pipelines() + + def set_pipelines(self): + models = self.valves.AZURE_OPENAI_MODELS.split(";") + model_names = self.valves.AZURE_OPENAI_MODEL_NAMES.split(";") + self.pipelines = [ + {"id": model, "name": name} for model, name in zip(models, model_names) + ] + print(f"azure_dalle_pipeline - models: {self.pipelines}") + + async def on_startup(self) -> None: + print(f"on_startup:{__name__}") + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + + async def on_valves_updated(self): + print(f"on_valves_updated:{__name__}") + self.set_pipelines() + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + print(f"pipe:{__name__}") + + headers = { + "api-key": self.valves.AZURE_OPENAI_API_KEY, + "Content-Type": "application/json", + } + + url = f"{self.valves.AZURE_OPENAI_ENDPOINT}/openai/deployments/{model_id}/images/generations?api-version={self.valves.AZURE_OPENAI_API_VERSION}" + + payload = { + "model": model_id, + "prompt": user_message, + "size": self.valves.IMAGE_SIZE, + "n": self.valves.NUM_IMAGES, + } + + try: + response = requests.post(url, json=payload, headers=headers) + response.raise_for_status() + data = response.json() + + message = "" + for image in data.get("data", []): + if "url" in image: + message += f"![image]({image['url']})\n" + + yield message + except Exception as e: + yield f"Error: {e} ({response.text if response else 'No response'})" diff --git a/openwebui/pipelines/examples/pipelines/providers/azure_deepseek_r1_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/azure_deepseek_r1_pipeline.py new file mode 100644 index 0000000..fc4c14e --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/azure_deepseek_r1_pipeline.py @@ -0,0 +1,99 @@ +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import requests +import os + + +class Pipeline: + class Valves(BaseModel): + # You can add your custom valves here. + AZURE_DEEPSEEKR1_API_KEY: str + AZURE_DEEPSEEKR1_ENDPOINT: str + AZURE_DEEPSEEKR1_API_VERSION: str + + def __init__(self): + self.type = "manifold" + self.name = "Azure " + self.valves = self.Valves( + **{ + "AZURE_DEEPSEEKR1_API_KEY": os.getenv("AZURE_DEEPSEEKR1_API_KEY", "your-azure-deepseek-r1-api-key-here"), + "AZURE_DEEPSEEKR1_ENDPOINT": os.getenv("AZURE_DEEPSEEKR1_ENDPOINT", "your-azure-deepseek-r1-endpoint-here"), + "AZURE_DEEPSEEKR1_API_VERSION": os.getenv("AZURE_DEEPSEEKR1_API_VERSION", "2024-05-01-preview"), + } + ) + self.set_pipelines() + pass + + def set_pipelines(self): + models = ['DeepSeek-R1'] + model_names = ['DeepSeek-R1'] + self.pipelines = [ + {"id": model, "name": name} for model, name in zip(models, model_names) + ] + print(f"azure_deepseek_r1_pipeline - models: {self.pipelines}") + pass + + async def on_valves_updated(self): + self.set_pipelines() + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + headers = { + "api-key": self.valves.AZURE_DEEPSEEKR1_API_KEY, + "Content-Type": "application/json", + } + + url = f"{self.valves.AZURE_DEEPSEEKR1_ENDPOINT}/models/chat/completions?api-version={self.valves.AZURE_DEEPSEEKR1_API_VERSION}" + + print(body) + + allowed_params = {'messages', 'temperature', 'role', 'content', 'contentPart', 'contentPartImage', + 'enhancements', 'dataSources', 'n', 'stream', 'stop', 'max_tokens', 'presence_penalty', + 'frequency_penalty', 'logit_bias', 'user', 'function_call', 'funcions', 'tools', + 'tool_choice', 'top_p', 'log_probs', 'top_logprobs', 'response_format', 'seed', 'model'} + # remap user field + if "user" in body and not isinstance(body["user"], str): + body["user"] = body["user"]["id"] if "id" in body["user"] else str(body["user"]) + # Fill in model field as per Azure's api requirements + body["model"] = model_id + filtered_body = {k: v for k, v in body.items() if k in allowed_params} + # log fields that were filtered out as a single line + if len(body) != len(filtered_body): + print(f"Dropped params: {', '.join(set(body.keys()) - set(filtered_body.keys()))}") + + try: + r = requests.post( + url=url, + json=filtered_body, + headers=headers, + stream=True, + ) + + r.raise_for_status() + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + if r: + text = r.text + return f"Error: {e} ({text})" + else: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/azure_jais_core42_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/azure_jais_core42_pipeline.py new file mode 100644 index 0000000..2b8e8a7 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/azure_jais_core42_pipeline.py @@ -0,0 +1,215 @@ +""" +title: Jais Azure Pipeline with Stream Handling Fix +author: Abdessalaam Al-Alestini +date: 2024-06-20 +version: 1.3 +license: MIT +description: A pipeline for generating text using the Jais model via Azure AI Inference API, with fixed stream handling. +About Jais: https://inceptionai.ai/jais/ +requirements: azure-ai-inference +environment_variables: AZURE_INFERENCE_CREDENTIAL, AZURE_INFERENCE_ENDPOINT, MODEL_ID +""" + +import os +import json +import logging +from typing import List, Union, Generator, Iterator, Tuple +from pydantic import BaseModel +from azure.ai.inference import ChatCompletionsClient +from azure.core.credentials import AzureKeyCredential +from azure.ai.inference.models import SystemMessage, UserMessage, AssistantMessage + +# Set up logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +def pop_system_message(messages: List[dict]) -> Tuple[str, List[dict]]: + """ + Extract the system message from the list of messages. + + Args: + messages (List[dict]): List of message dictionaries. + + Returns: + Tuple[str, List[dict]]: A tuple containing the system message (or empty string) and the updated list of messages. + """ + system_message = "" + updated_messages = [] + + for message in messages: + if message['role'] == 'system': + system_message = message['content'] + else: + updated_messages.append(message) + + return system_message, updated_messages + + +class Pipeline: + + class Valves(BaseModel): + AZURE_INFERENCE_CREDENTIAL: str = "" + AZURE_INFERENCE_ENDPOINT: str = "" + MODEL_ID: str = "jais-30b-chat" + + def __init__(self): + self.type = "manifold" + self.id = "jais-azure" + self.name = "jais-azure/" + + self.valves = self.Valves( + **{ + "AZURE_INFERENCE_CREDENTIAL": + os.getenv("AZURE_INFERENCE_CREDENTIAL", + "your-azure-inference-key-here"), + "AZURE_INFERENCE_ENDPOINT": + os.getenv("AZURE_INFERENCE_ENDPOINT", + "your-azure-inference-endpoint-here"), + "MODEL_ID": + os.getenv("MODEL_ID", "jais-30b-chat"), + }) + self.update_client() + + def update_client(self): + self.client = ChatCompletionsClient( + endpoint=self.valves.AZURE_INFERENCE_ENDPOINT, + credential=AzureKeyCredential( + self.valves.AZURE_INFERENCE_CREDENTIAL)) + + def get_jais_models(self): + return [ + { + "id": "jais-30b-chat", + "name": "Jais 30B Chat" + }, + ] + + async def on_startup(self): + logger.info(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + logger.info(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + self.update_client() + + def pipelines(self) -> List[dict]: + return self.get_jais_models() + + def pipe(self, user_message: str, model_id: str, messages: List[dict], + body: dict) -> Union[str, Generator, Iterator]: + try: + logger.debug( + f"Received request - user_message: {user_message}, model_id: {model_id}" + ) + logger.debug(f"Messages: {json.dumps(messages, indent=2)}") + logger.debug(f"Body: {json.dumps(body, indent=2)}") + + # Remove unnecessary keys + for key in ['user', 'chat_id', 'title']: + body.pop(key, None) + + system_message, messages = pop_system_message(messages) + + # Prepare messages for Jais + jais_messages = [SystemMessage( + content=system_message)] if system_message else [] + jais_messages += [ + UserMessage(content=msg['content']) if msg['role'] == 'user' + else SystemMessage(content=msg['content']) if msg['role'] + == 'system' else AssistantMessage(content=msg['content']) + for msg in messages + ] + + # Prepare the payload + allowed_params = { + 'temperature', 'max_tokens', 'presence_penalty', + 'frequency_penalty', 'top_p' + } + filtered_body = { + k: v + for k, v in body.items() if k in allowed_params + } + + logger.debug(f"Prepared Jais messages: {jais_messages}") + logger.debug(f"Filtered body: {filtered_body}") + + is_stream = body.get("stream", False) + if is_stream: + return self.stream_response(jais_messages, filtered_body) + else: + return self.get_completion(jais_messages, filtered_body) + except Exception as e: + logger.error(f"Error in pipe: {str(e)}", exc_info=True) + return json.dumps({"error": str(e)}) + + def stream_response(self, jais_messages: List[Union[SystemMessage, UserMessage, AssistantMessage]], params: dict) -> str: + try: + complete_response = "" + response = self.client.complete(messages=jais_messages, + model=self.valves.MODEL_ID, + stream=True, + **params) + for update in response: + if update.choices: + delta_content = update.choices[0].delta.content + if delta_content: + complete_response += delta_content + return complete_response + except Exception as e: + logger.error(f"Error in stream_response: {str(e)}", exc_info=True) + return json.dumps({"error": str(e)}) + + def get_completion(self, jais_messages: List[Union[SystemMessage, UserMessage, AssistantMessage]], params: dict) -> str: + try: + response = self.client.complete(messages=jais_messages, + model=self.valves.MODEL_ID, + **params) + if response.choices: + result = response.choices[0].message.content + logger.debug(f"Completion result: {result}") + return result + else: + logger.warning("No choices in completion response") + return "" + except Exception as e: + logger.error(f"Error in get_completion: {str(e)}", exc_info=True) + return json.dumps({"error": str(e)}) + + +# TEST CASE TO RUN THE PIPELINE +if __name__ == "__main__": + pipeline = Pipeline() + + messages = [{ + "role": "user", + "content": "How many languages are in the world?" + }] + body = { + "temperature": 0.5, + "max_tokens": 150, + "presence_penalty": 0.1, + "frequency_penalty": 0.8, + "stream": True # Change to True to test streaming + } + + result = pipeline.pipe(user_message="How many languages are in the world?", + model_id="jais-30b-chat", + messages=messages, + body=body) + + # Handle streaming result + if isinstance(result, str): + content = json.dumps({"content": result}, ensure_ascii=False) + print(content) + else: + complete_response = "" + for part in result: + content_delta = json.loads(part).get("delta") + if content_delta: + complete_response += content_delta + + print(json.dumps({"content": complete_response}, ensure_ascii=False)) diff --git a/openwebui/pipelines/examples/pipelines/providers/azure_openai_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/azure_openai_manifold_pipeline.py new file mode 100644 index 0000000..6f77a44 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/azure_openai_manifold_pipeline.py @@ -0,0 +1,99 @@ +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import requests +import os + + +class Pipeline: + class Valves(BaseModel): + # You can add your custom valves here. + AZURE_OPENAI_API_KEY: str + AZURE_OPENAI_ENDPOINT: str + AZURE_OPENAI_API_VERSION: str + AZURE_OPENAI_MODELS: str + AZURE_OPENAI_MODEL_NAMES: str + + def __init__(self): + self.type = "manifold" + self.name = "Azure OpenAI: " + self.valves = self.Valves( + **{ + "AZURE_OPENAI_API_KEY": os.getenv("AZURE_OPENAI_API_KEY", "your-azure-openai-api-key-here"), + "AZURE_OPENAI_ENDPOINT": os.getenv("AZURE_OPENAI_ENDPOINT", "your-azure-openai-endpoint-here"), + "AZURE_OPENAI_API_VERSION": os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-01"), + "AZURE_OPENAI_MODELS": os.getenv("AZURE_OPENAI_MODELS", "gpt-35-turbo;gpt-4o"), + "AZURE_OPENAI_MODEL_NAMES": os.getenv("AZURE_OPENAI_MODEL_NAMES", "GPT-35 Turbo;GPT-4o"), + } + ) + self.set_pipelines() + pass + + def set_pipelines(self): + models = self.valves.AZURE_OPENAI_MODELS.split(";") + model_names = self.valves.AZURE_OPENAI_MODEL_NAMES.split(";") + self.pipelines = [ + {"id": model, "name": name} for model, name in zip(models, model_names) + ] + print(f"azure_openai_manifold_pipeline - models: {self.pipelines}") + pass + + async def on_valves_updated(self): + self.set_pipelines() + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + headers = { + "api-key": self.valves.AZURE_OPENAI_API_KEY, + "Content-Type": "application/json", + } + + url = f"{self.valves.AZURE_OPENAI_ENDPOINT}/openai/deployments/{model_id}/chat/completions?api-version={self.valves.AZURE_OPENAI_API_VERSION}" + + allowed_params = {'messages', 'temperature', 'role', 'content', 'contentPart', 'contentPartImage', + 'enhancements', 'dataSources', 'n', 'stream', 'stop', 'max_tokens', 'presence_penalty', + 'frequency_penalty', 'logit_bias', 'user', 'function_call', 'funcions', 'tools', + 'tool_choice', 'top_p', 'log_probs', 'top_logprobs', 'response_format', 'seed'} + # remap user field + if "user" in body and not isinstance(body["user"], str): + body["user"] = body["user"]["id"] if "id" in body["user"] else str(body["user"]) + filtered_body = {k: v for k, v in body.items() if k in allowed_params} + # log fields that were filtered out as a single line + if len(body) != len(filtered_body): + print(f"Dropped params: {', '.join(set(body.keys()) - set(filtered_body.keys()))}") + + try: + r = requests.post( + url=url, + json=filtered_body, + headers=headers, + stream=True, + ) + + r.raise_for_status() + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + if r: + text = r.text + return f"Error: {e} ({text})" + else: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/azure_openai_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/azure_openai_pipeline.py new file mode 100644 index 0000000..bb4e6e7 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/azure_openai_pipeline.py @@ -0,0 +1,90 @@ +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import requests +import os + + +class Pipeline: + class Valves(BaseModel): + # You can add your custom valves here. + AZURE_OPENAI_API_KEY: str + AZURE_OPENAI_ENDPOINT: str + AZURE_OPENAI_DEPLOYMENT_NAME: str + AZURE_OPENAI_API_VERSION: str + + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "azure_openai_pipeline" + self.name = "Azure OpenAI Pipeline" + self.valves = self.Valves( + **{ + "AZURE_OPENAI_API_KEY": os.getenv("AZURE_OPENAI_API_KEY", "your-azure-openai-api-key-here"), + "AZURE_OPENAI_ENDPOINT": os.getenv("AZURE_OPENAI_ENDPOINT", "your-azure-openai-endpoint-here"), + "AZURE_OPENAI_DEPLOYMENT_NAME": os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "your-deployment-name-here"), + "AZURE_OPENAI_API_VERSION": os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-01"), + } + ) + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + headers = { + "api-key": self.valves.AZURE_OPENAI_API_KEY, + "Content-Type": "application/json", + } + + url = f"{self.valves.AZURE_OPENAI_ENDPOINT}/openai/deployments/{self.valves.AZURE_OPENAI_DEPLOYMENT_NAME}/chat/completions?api-version={self.valves.AZURE_OPENAI_API_VERSION}" + + allowed_params = {'messages', 'temperature', 'role', 'content', 'contentPart', 'contentPartImage', + 'enhancements', 'data_sources', 'n', 'stream', 'stop', 'max_tokens', 'presence_penalty', + 'frequency_penalty', 'logit_bias', 'user', 'function_call', 'functions', 'tools', + 'tool_choice', 'top_p', 'log_probs', 'top_logprobs', 'response_format', 'seed'} + # remap user field + if "user" in body and not isinstance(body["user"], str): + body["user"] = body["user"]["id"] if "id" in body["user"] else str(body["user"]) + filtered_body = {k: v for k, v in body.items() if k in allowed_params} + # log fields that were filtered out as a single line + if len(body) != len(filtered_body): + print(f"Dropped params: {', '.join(set(body.keys()) - set(filtered_body.keys()))}") + + # Initialize the response variable to None. + r = None + try: + r = requests.post( + url=url, + json=filtered_body, + headers=headers, + stream=True, + ) + + r.raise_for_status() + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + if r: + text = r.text + return f"Error: {e} ({text})" + else: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/cloudflare_ai_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/cloudflare_ai_pipeline.py new file mode 100644 index 0000000..3bbcadc --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/cloudflare_ai_pipeline.py @@ -0,0 +1,83 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import os +import requests + + +class Pipeline: + class Valves(BaseModel): + CLOUDFLARE_ACCOUNT_ID: str = "" + CLOUDFLARE_API_KEY: str = "" + CLOUDFLARE_MODEL: str = "" + pass + + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "openai_pipeline" + self.name = "Cloudlfare AI" + self.valves = self.Valves( + **{ + "CLOUDFLARE_ACCOUNT_ID": os.getenv( + "CLOUDFLARE_ACCOUNT_ID", + "your-account-id", + ), + "CLOUDFLARE_API_KEY": os.getenv( + "CLOUDFLARE_API_KEY", "your-cloudflare-api-key" + ), + "CLOUDFLARE_MODEL": os.getenv( + "CLOUDFLARE_MODELS", + "@cf/meta/llama-3.1-8b-instruct", + ), + } + ) + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + headers = {} + headers["Authorization"] = f"Bearer {self.valves.CLOUDFLARE_API_KEY}" + headers["Content-Type"] = "application/json" + + payload = {**body, "model": self.valves.CLOUDFLARE_MODEL} + + if "user" in payload: + del payload["user"] + if "chat_id" in payload: + del payload["chat_id"] + if "title" in payload: + del payload["title"] + + try: + r = requests.post( + url=f"https://api.cloudflare.com/client/v4/accounts/{self.valves.CLOUDFLARE_ACCOUNT_ID}/ai/v1/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/cohere_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/cohere_manifold_pipeline.py new file mode 100644 index 0000000..61fcf8b --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/cohere_manifold_pipeline.py @@ -0,0 +1,163 @@ +""" +title: Cohere Manifold Pipeline +author: justinh-rahb +date: 2024-05-28 +version: 1.0 +license: MIT +description: A pipeline for generating text using the Anthropic API. +requirements: requests +environment_variables: COHERE_API_KEY +""" + +import os +import json +from schemas import OpenAIChatMessage +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import requests + + +class Pipeline: + class Valves(BaseModel): + COHERE_API_BASE_URL: str = "https://api.cohere.com/v1" + COHERE_API_KEY: str = "" + + def __init__(self): + self.type = "manifold" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + + self.id = "cohere" + + self.name = "cohere/" + + self.valves = self.Valves( + **{"COHERE_API_KEY": os.getenv("COHERE_API_KEY", "your-api-key-here")} + ) + + self.pipelines = self.get_cohere_models() + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + + self.pipelines = self.get_cohere_models() + + pass + + def get_cohere_models(self): + if self.valves.COHERE_API_KEY: + try: + headers = {} + headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}" + headers["Content-Type"] = "application/json" + + r = requests.get( + f"{self.valves.COHERE_API_BASE_URL}/models", headers=headers + ) + + models = r.json() + return [ + { + "id": model["name"], + "name": model["name"] if "name" in model else model["name"], + } + for model in models["models"] + ] + except Exception as e: + + print(f"Error: {e}") + return [ + { + "id": self.id, + "name": "Could not fetch models from Cohere, please update the API Key in the valves.", + }, + ] + else: + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + try: + if body.get("stream", False): + return self.stream_response(user_message, model_id, messages, body) + else: + return self.get_completion(user_message, model_id, messages, body) + except Exception as e: + return f"Error: {e}" + + def stream_response( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Generator: + + headers = {} + headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}" + headers["Content-Type"] = "application/json" + + r = requests.post( + url=f"{self.valves.COHERE_API_BASE_URL}/chat", + json={ + "model": model_id, + "chat_history": [ + { + "role": "USER" if message["role"] == "user" else "CHATBOT", + "message": message["content"], + } + for message in messages[:-1] + ], + "message": user_message, + "stream": True, + }, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + for line in r.iter_lines(): + if line: + try: + line = json.loads(line) + if line["event_type"] == "text-generation": + yield line["text"] + except: + pass + + def get_completion( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> str: + headers = {} + headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}" + headers["Content-Type"] = "application/json" + + r = requests.post( + url=f"{self.valves.COHERE_API_BASE_URL}/chat", + json={ + "model": model_id, + "chat_history": [ + { + "role": "USER" if message["role"] == "user" else "CHATBOT", + "message": message["content"], + } + for message in messages[:-1] + ], + "message": user_message, + }, + headers=headers, + ) + + r.raise_for_status() + data = r.json() + + return data["text"] if "text" in data else "No response from Cohere." diff --git a/openwebui/pipelines/examples/pipelines/providers/deepseek_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/deepseek_manifold_pipeline.py new file mode 100644 index 0000000..a8b1c49 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/deepseek_manifold_pipeline.py @@ -0,0 +1,150 @@ +""" +title: DeepSeek Manifold Pipeline +author: Mohammed El-Beltagy +date: 2025-01-20 +version: 1.4 +license: MIT +description: A pipeline for generating text using the DeepSeeks API. +requirements: requests, sseclient-py +environment_variables: DEEPSEEK_API_KEY +""" + + +import os +import requests +import json +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import sseclient + +from utils.pipelines.main import pop_system_message + + +class Pipeline: + class Valves(BaseModel): + DEEPSEEK_API_KEY: str = "" + + def __init__(self): + self.type = "manifold" + self.id = "deepseek" + self.name = "deepseek/" + + self.valves = self.Valves( + **{"DEEPSEEK_API_KEY": os.getenv("DEEPSEEK_API_KEY", "your-api-key-here")} + ) + self.url = 'https://api.deepseek.com/chat/completions' + self.update_headers() + + def update_headers(self): + self.headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.valves.DEEPSEEK_API_KEY}' + } + + def get_deepseek_models(self): + return [ + {"id": "deepseek-chat", "name": "DeepSeek Chat"}, + {"id": "deepseek-reasoner", "name": "DeepSeek R1"}, + ] + + async def on_startup(self): + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + self.update_headers() + + def pipelines(self) -> List[dict]: + return self.get_deepseek_models() + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + try: + # Remove unnecessary keys + for key in ['user', 'chat_id', 'title']: + body.pop(key, None) + + system_message, messages = pop_system_message(messages) + + # Process messages for DeepSeek format + processed_messages = [] + for message in messages: + if isinstance(message.get("content"), list): + # DeepSeek currently doesn't support multi-modal inputs + # Combine all text content + text_content = " ".join( + item["text"] for item in message["content"] + if item["type"] == "text" + ) + processed_messages.append({ + "role": message["role"], + "content": text_content + }) + else: + processed_messages.append({ + "role": message["role"], + "content": message.get("content", "") + }) + + # Add system message if present + if system_message: + processed_messages.insert(0, { + "role": "system", + "content": str(system_message) + }) + + # Prepare the payload for DeepSeek API + payload = { + "model": model_id, + "messages": processed_messages, + "max_tokens": body.get("max_tokens", 4096), + "temperature": body.get("temperature", 0.8), + "top_p": body.get("top_p", 0.9), + "stream": body.get("stream", False) + } + + # Add optional parameters if present + if "stop" in body: + payload["stop"] = body["stop"] + + if body.get("stream", False): + return self.stream_response(payload) + else: + return self.get_completion(payload) + except Exception as e: + return f"Error: {e}" + + def stream_response(self, payload: dict) -> Generator: + response = requests.post(self.url, headers=self.headers, json=payload, stream=True) + + if response.status_code == 200: + client = sseclient.SSEClient(response) + for event in client.events(): + try: + data = json.loads(event.data) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + if "content" in delta: + yield delta["content"] + if data["choices"][0].get("finish_reason") is not None: + break + except json.JSONDecodeError: + print(f"Failed to parse JSON: {event.data}") + except KeyError as e: + print(f"Unexpected data structure: {e}") + print(f"Full data: {data}") + else: + raise Exception(f"Error: {response.status_code} - {response.text}") + + def get_completion(self, payload: dict) -> str: + response = requests.post(self.url, headers=self.headers, json=payload) + if response.status_code == 200: + res = response.json() + return res["choices"][0]["message"]["content"] if "choices" in res else "" + else: + raise Exception(f"Error: {response.status_code} - {response.text}") diff --git a/openwebui/pipelines/examples/pipelines/providers/google_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/google_manifold_pipeline.py new file mode 100644 index 0000000..d5d500b --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/google_manifold_pipeline.py @@ -0,0 +1,210 @@ +""" +title: Google GenAI Manifold Pipeline +author: Marc Lopez (refactor by justinh-rahb) +date: 2024-06-06 +version: 1.3 +license: MIT +description: A pipeline for generating text using Google's GenAI models in Open-WebUI. +requirements: google-genai +environment_variables: GOOGLE_API_KEY +""" + +from typing import List, Union, Iterator +import os + +from pydantic import BaseModel, Field + +from google import genai +from google.genai import types +from PIL import Image +from io import BytesIO +import base64 + + +class Pipeline: + """Google GenAI pipeline""" + + class Valves(BaseModel): + """Options to change from the WebUI""" + + GOOGLE_API_KEY: str = Field(default="",description="Google Generative AI API key") + USE_PERMISSIVE_SAFETY: bool = Field(default=False,description="Use permissive safety settings") + GENERATE_IMAGE: bool = Field(default=False,description="Allow image generation") + + def __init__(self): + self.type = "manifold" + self.id = "google_genai" + self.name = "Google: " + + self.valves = self.Valves(**{ + "GOOGLE_API_KEY": os.getenv("GOOGLE_API_KEY", ""), + "USE_PERMISSIVE_SAFETY": False, + "GENERATE_IMAGE": False + }) + self.pipelines = [] + + if self.valves.GOOGLE_API_KEY: + self.update_pipelines() + + async def on_startup(self) -> None: + """This function is called when the server is started.""" + + print(f"on_startup:{__name__}") + if self.valves.GOOGLE_API_KEY: + self.update_pipelines() + + async def on_shutdown(self) -> None: + """This function is called when the server is stopped.""" + + print(f"on_shutdown:{__name__}") + + async def on_valves_updated(self) -> None: + """This function is called when the valves are updated.""" + + print(f"on_valves_updated:{__name__}") + if self.valves.GOOGLE_API_KEY: + self.update_pipelines() + + def update_pipelines(self) -> None: + """Update the available models from Google GenAI""" + + if self.valves.GOOGLE_API_KEY: + client = genai.Client(api_key=self.valves.GOOGLE_API_KEY) + try: + models = client.models.list() + self.pipelines = [ + { + "id": model.name[7:], # the "models/" part messeses up the URL + "name": model.display_name, + } + for model in models + if "generateContent" in model.supported_actions + if model.name[:7] == "models/" + ] + except Exception: + self.pipelines = [ + { + "id": "error", + "name": "Could not fetch models from Google, please update the API Key in the valves.", + } + ] + else: + self.pipelines = [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Iterator]: + if not self.valves.GOOGLE_API_KEY: + return "Error: GOOGLE_API_KEY is not set" + + try: + client = genai.Client(api_key=self.valves.GOOGLE_API_KEY) + + if model_id.startswith("google_genai."): + model_id = model_id[12:] + model_id = model_id.lstrip(".") + + if not (model_id.startswith("gemini-") or model_id.startswith("learnlm-") or model_id.startswith("gemma-")): + return f"Error: Invalid model name format: {model_id}" + + print(f"Pipe function called for model: {model_id}") + print(f"Stream mode: {body.get('stream', False)}") + + system_message = next((msg["content"] for msg in messages if msg["role"] == "system"), None) + + contents = [] + for message in messages: + if message["role"] != "system": + if isinstance(message.get("content"), list): + parts = [] + for content in message["content"]: + if content["type"] == "text": + parts.append({"text": content["text"]}) + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + if image_url.startswith("data:image"): + image_data = image_url.split(",")[1] + parts.append({"inline_data": {"mime_type": "image/jpeg", "data": image_data}}) + else: + parts.append({"image_url": image_url}) + contents.append({"role": message["role"], "parts": parts}) + else: + contents.append({ + "role": "user" if message["role"] == "user" else "model", + "parts": [{"text": message["content"]}] + }) + print(f"{contents}") + + generation_config = { + "temperature": body.get("temperature", 0.7), + "top_p": body.get("top_p", 0.9), + "top_k": body.get("top_k", 40), + "max_output_tokens": body.get("max_tokens", 8192), + "stop_sequences": body.get("stop", []), + "response_modalities": ['Text'] + } + + if self.valves.GENERATE_IMAGE and model_id.startswith("gemini-2.0-flash-exp"): + generation_config["response_modalities"].append("Image") + + if self.valves.USE_PERMISSIVE_SAFETY: + safety_settings = [ + types.SafetySetting(category='HARM_CATEGORY_HARASSMENT', threshold='OFF'), + types.SafetySetting(category='HARM_CATEGORY_HATE_SPEECH', threshold='OFF'), + types.SafetySetting(category='HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold='OFF'), + types.SafetySetting(category='HARM_CATEGORY_DANGEROUS_CONTENT', threshold='OFF'), + types.SafetySetting(category='HARM_CATEGORY_CIVIC_INTEGRITY', threshold='OFF') + ] + generation_config = types.GenerateContentConfig(**generation_config, safety_settings=safety_settings) + else: + generation_config = types.GenerateContentConfig(**generation_config) + + if system_message: + contents.insert(0, {"role": "user", "parts": [{"text": f"System: {system_message}"}]}) + + if body.get("stream", False): + response = client.models.generate_content_stream( + model = model_id, + contents = contents, + config = generation_config, + ) + return self.stream_response(response) + else: + response = client.models.generate_content( + model = model_id, + contents = contents, + config = generation_config, + ) + for part in response.candidates[0].content.parts: + if part.text is not None: + return part.text + elif part.inline_data is not None: + try: + image_data = base64.b64decode(part.inline_data.data) + image = Image.open(BytesIO((image_data))) + content_type = part.inline_data.mime_type + return "Image not supported yet." + except Exception as e: + print(f"Error processing image: {e}") + return "Error processing image." + + except Exception as e: + print(f"Error generating content: {e}") + return f"{e}" + + def stream_response(self, response): + for chunk in response: + for candidate in chunk.candidates: + if candidate.content.parts is not None: + for part in candidate.content.parts: + if part.text is not None: + yield chunk.text + elif part.inline_data is not None: + try: + image_data = base64.b64decode(part.inline_data.data) + image = Image.open(BytesIO(image_data)) + content_type = part.inline_data.mime_type + yield "Image not supported yet." + except Exception as e: + print(f"Error processing image: {e}") + yield "Error processing image." diff --git a/openwebui/pipelines/examples/pipelines/providers/google_vertexai_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/google_vertexai_manifold_pipeline.py new file mode 100644 index 0000000..2d77d28 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/google_vertexai_manifold_pipeline.py @@ -0,0 +1,232 @@ +""" +title: Google GenAI (Vertex AI) Manifold Pipeline +author: Hiromasa Kakehashi & Olv Grolle +date: 2024-09-19 +version: 1.0 +license: MIT +description: A pipeline for generating text using Google's GenAI models in Open-WebUI. +requirements: vertexai +environment_variables: GOOGLE_PROJECT_ID, GOOGLE_CLOUD_REGION +usage_instructions: + To use Gemini with the Vertex AI API, a service account with the appropriate role (e.g., `roles/aiplatform.user`) is required. + - For deployment on Google Cloud: Associate the service account with the deployment. + - For use outside of Google Cloud: Set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of the service account key file. +""" + +import os +import base64 +from typing import Iterator, List, Union + +import vertexai +from pydantic import BaseModel, Field +from vertexai.generative_models import ( + Content, + GenerationConfig, + GenerativeModel, + HarmBlockThreshold, + HarmCategory, + Part, +) + + +class Pipeline: + """Google GenAI pipeline""" + + class Valves(BaseModel): + """Options to change from the WebUI""" + + GOOGLE_PROJECT_ID: str = "" + GOOGLE_CLOUD_REGION: str = "" + USE_PERMISSIVE_SAFETY: bool = Field(default=False) + + def __init__(self): + self.type = "manifold" + self.name = "VertexAI: " + + self.valves = self.Valves( + **{ + "GOOGLE_PROJECT_ID": os.getenv("GOOGLE_PROJECT_ID", ""), + "GOOGLE_CLOUD_REGION": os.getenv("GOOGLE_CLOUD_REGION", ""), + "USE_PERMISSIVE_SAFETY": False, + } + ) + self.pipelines = [ + + # Gemini 2.0 models + {"id": "gemini-2.0-flash-lite", "name": "Gemini 2.0 Flash-Lite"}, + {"id": "gemini-2.0-flash", "name": "Gemini 2.0 Flash"}, + # Gemini 2.5 models + {"id": "gemini-2.5-flash-lite", "name": "Gemini 2.5 Flash-Lite"}, + {"id": "gemini-2.5-flash", "name": "Gemini 2.5 Flash"}, + {"id": "gemini-2.5-pro", "name": "Gemini 2.5 Pro "}, + + ] + + async def on_startup(self) -> None: + """This function is called when the server is started.""" + + print(f"on_startup:{__name__}") + vertexai.init( + project=self.valves.GOOGLE_PROJECT_ID, + location=self.valves.GOOGLE_CLOUD_REGION, + ) + + async def on_shutdown(self) -> None: + """This function is called when the server is stopped.""" + print(f"on_shutdown:{__name__}") + + async def on_valves_updated(self) -> None: + """This function is called when the valves are updated.""" + print(f"on_valves_updated:{__name__}") + vertexai.init( + project=self.valves.GOOGLE_PROJECT_ID, + location=self.valves.GOOGLE_CLOUD_REGION, + ) + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Iterator]: + try: + if not (model_id.startswith("gemini-") or model_id.startswith("gemma-")): + return f"Error: Invalid model name format: {model_id}" + + print(f"Pipe function called for model: {model_id}") + print(f"Stream mode: {body.get('stream', False)}") + print(f"Received {len(messages)} messages from OpenWebUI") + + # Debug: Log message structure + for i, msg in enumerate(messages): + print(f"Message {i}: role={msg.get('role')}, content type={type(msg.get('content'))}") + if isinstance(msg.get('content'), list): + for j, content_part in enumerate(msg['content']): + print(f" Part {j}: type={content_part.get('type')}") + if content_part.get('type') == 'image_url': + img_url = content_part.get('image_url', {}).get('url', '') + print(f" Image URL prefix: {img_url[:50]}...") + + system_message = next( + (msg["content"] for msg in messages if msg["role"] == "system"), None + ) + + model = GenerativeModel( + model_name=model_id, + system_instruction=system_message, + ) + + if body.get("title", False): # If chat title generation is requested + contents = [Content(role="user", parts=[Part.from_text(user_message)])] + print("Title generation mode - using simple text content") + else: + contents = self.build_conversation_history(messages) + + # Log what we're sending to Vertex AI + print(f"Sending {len(contents)} messages to Vertex AI:") + for i, content in enumerate(contents): + print(f" Message {i}: role={content.role}, parts={len(content.parts)}") + for j, part in enumerate(content.parts): + if hasattr(part, '_raw_data') and part._raw_data: + print(f" Part {j}: Image data ({len(part._raw_data)} bytes)") + else: + part_text = str(part)[:100] if str(part) else "No text" + print(f" Part {j}: Text - {part_text}...") + + generation_config = GenerationConfig( + temperature=body.get("temperature", 0.7), + top_p=body.get("top_p", 0.9), + top_k=body.get("top_k", 40), + max_output_tokens=body.get("max_tokens", 8192), + stop_sequences=body.get("stop", []), + ) + + if self.valves.USE_PERMISSIVE_SAFETY: + safety_settings = { + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + } + else: + safety_settings = body.get("safety_settings") + + print("Calling Vertex AI generate_content...") + response = model.generate_content( + contents, + stream=body.get("stream", False), + generation_config=generation_config, + safety_settings=safety_settings, + ) + + if body.get("stream", False): + return self.stream_response(response) + else: + return response.text + + except Exception as e: + print(f"Error generating content: {e}") + return f"An error occurred: {str(e)}" + + def stream_response(self, response): + for chunk in response: + if chunk.text: + print(f"Chunk: {chunk.text}") + yield chunk.text + + def build_conversation_history(self, messages: List[dict]) -> List[Content]: + contents = [] + + for message in messages: + if message["role"] == "system": + continue + + parts = [] + + if isinstance(message.get("content"), list): + print(f"Processing multi-part message with {len(message['content'])} parts") + for content in message["content"]: + print(f"Processing content type: {content.get('type', 'unknown')}") + if content["type"] == "text": + parts.append(Part.from_text(content["text"])) + print(f"Added text part: {content['text'][:50]}...") + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + print(f"Processing image URL (first 50 chars): {image_url[:50]}...") + if image_url.startswith("data:image"): + try: + # Split the data URL to get mime type and base64 data + header, image_data = image_url.split(',', 1) + mime_type = header.split(':')[1].split(';')[0] + print(f"Detected image MIME type: {mime_type}") + + # Validate supported image formats + supported_formats = ['image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp'] + if mime_type not in supported_formats: + print(f"ERROR: Unsupported image format: {mime_type}") + continue + + # Decode the base64 image data + decoded_image_data = base64.b64decode(image_data) + print(f"Successfully decoded image data: {len(decoded_image_data)} bytes") + + # Create the Part object with the image data + image_part = Part.from_data(decoded_image_data, mime_type=mime_type) + parts.append(image_part) + print(f"Successfully added image part to conversation") + except Exception as e: + print(f"ERROR processing image: {e}") + import traceback + traceback.print_exc() + continue + else: + # Handle image URLs + print(f"Processing external image URL: {image_url}") + parts.append(Part.from_uri(image_url)) + else: + parts = [Part.from_text(message["content"])] + print(f"Added simple text message: {message['content'][:50]}...") + + role = "user" if message["role"] == "user" else "model" + print(f"Creating Content with role='{role}' and {len(parts)} parts") + contents.append(Content(role=role, parts=parts)) + + print(f"Built conversation history with {len(contents)} messages") + return contents diff --git a/openwebui/pipelines/examples/pipelines/providers/groq_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/groq_manifold_pipeline.py new file mode 100644 index 0000000..717f738 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/groq_manifold_pipeline.py @@ -0,0 +1,122 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel + +import os +import requests + + +class Pipeline: + class Valves(BaseModel): + GROQ_API_BASE_URL: str = "https://api.groq.com/openai/v1" + GROQ_API_KEY: str = "" + pass + + def __init__(self): + self.type = "manifold" + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + self.id = "groq" + self.name = "Groq: " + + self.valves = self.Valves( + **{ + "GROQ_API_KEY": os.getenv( + "GROQ_API_KEY", "your-groq-api-key-here" + ) + } + ) + + self.pipelines = self.get_models() + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + self.pipelines = self.get_models() + pass + + def get_models(self): + if self.valves.GROQ_API_KEY: + try: + headers = {} + headers["Authorization"] = f"Bearer {self.valves.GROQ_API_KEY}" + headers["Content-Type"] = "application/json" + + r = requests.get( + f"{self.valves.GROQ_API_BASE_URL}/models", headers=headers + ) + + models = r.json() + return [ + { + "id": model["id"], + "name": model["name"] if "name" in model else model["id"], + } + for model in models["data"] + ] + + except Exception as e: + + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from Groq, please update the API Key in the valves.", + }, + ] + else: + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + headers = {} + headers["Authorization"] = f"Bearer {self.valves.GROQ_API_KEY}" + headers["Content-Type"] = "application/json" + + payload = {**body, "model": model_id} + + if "user" in payload: + del payload["user"] + if "chat_id" in payload: + del payload["chat_id"] + if "title" in payload: + del payload["title"] + + print(payload) + + try: + r = requests.post( + url=f"{self.valves.GROQ_API_BASE_URL}/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/providers/litellm_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/litellm_manifold_pipeline.py new file mode 100644 index 0000000..904a9f0 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/litellm_manifold_pipeline.py @@ -0,0 +1,135 @@ +""" +title: LiteLLM Manifold Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0.1 +license: MIT +description: A manifold pipeline that uses LiteLLM. +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import os + + +class Pipeline: + + class Valves(BaseModel): + LITELLM_BASE_URL: str = "" + LITELLM_API_KEY: str = "" + LITELLM_PIPELINE_DEBUG: bool = False + + def __init__(self): + # You can also set the pipelines that are available in this pipeline. + # Set manifold to True if you want to use this pipeline as a manifold. + # Manifold pipelines can have multiple pipelines. + self.type = "manifold" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "litellm_manifold" + + # Optionally, you can set the name of the manifold pipeline. + self.name = "LiteLLM: " + + # Initialize rate limits + self.valves = self.Valves( + **{ + "LITELLM_BASE_URL": os.getenv( + "LITELLM_BASE_URL", "http://localhost:4001" + ), + "LITELLM_API_KEY": os.getenv("LITELLM_API_KEY", "your-api-key-here"), + "LITELLM_PIPELINE_DEBUG": os.getenv("LITELLM_PIPELINE_DEBUG", False), + } + ) + # Get models on initialization + self.pipelines = self.get_litellm_models() + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + # Get models on startup + self.pipelines = self.get_litellm_models() + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + + self.pipelines = self.get_litellm_models() + pass + + def get_litellm_models(self): + + headers = {} + if self.valves.LITELLM_API_KEY: + headers["Authorization"] = f"Bearer {self.valves.LITELLM_API_KEY}" + + if self.valves.LITELLM_BASE_URL: + try: + r = requests.get( + f"{self.valves.LITELLM_BASE_URL}/v1/models", headers=headers + ) + models = r.json() + return [ + { + "id": model["id"], + "name": model["name"] if "name" in model else model["id"], + } + for model in models["data"] + ] + except Exception as e: + print(f"Error fetching models from LiteLLM: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from LiteLLM, please update the URL in the valves.", + }, + ] + else: + print("LITELLM_BASE_URL not set. Please configure it in the valves.") + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + if "user" in body: + print("######################################") + print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})') + print(f"# Message: {user_message}") + print("######################################") + + headers = {} + if self.valves.LITELLM_API_KEY: + headers["Authorization"] = f"Bearer {self.valves.LITELLM_API_KEY}" + + try: + payload = {**body, "model": model_id, "user": body["user"]["id"]} + payload.pop("chat_id", None) + payload.pop("user", None) + payload.pop("title", None) + + r = requests.post( + url=f"{self.valves.LITELLM_BASE_URL}/v1/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/litellm_subprocess_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/litellm_subprocess_manifold_pipeline.py new file mode 100644 index 0000000..99b778a --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/litellm_subprocess_manifold_pipeline.py @@ -0,0 +1,211 @@ +""" +title: LiteLLM Subprocess Manifold Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A manifold pipeline that uses LiteLLM as a subprocess. +requirements: yaml, litellm[proxy] +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests + + +import os +import asyncio +import subprocess +import yaml + + +class Pipeline: + class Valves(BaseModel): + LITELLM_CONFIG_DIR: str = "./litellm/config.yaml" + LITELLM_PROXY_PORT: int = 4001 + LITELLM_PROXY_HOST: str = "127.0.0.1" + litellm_config: dict = {} + + def __init__(self): + # You can also set the pipelines that are available in this pipeline. + # Set manifold to True if you want to use this pipeline as a manifold. + # Manifold pipelines can have multiple pipelines. + self.type = "manifold" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "litellm_subprocess_manifold" + + # Optionally, you can set the name of the manifold pipeline. + self.name = "LiteLLM: " + + # Initialize Valves + self.valves = self.Valves(**{"LITELLM_CONFIG_DIR": f"./litellm/config.yaml"}) + self.background_process = None + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + + # Check if the config file exists + if not os.path.exists(self.valves.LITELLM_CONFIG_DIR): + with open(self.valves.LITELLM_CONFIG_DIR, "w") as file: + yaml.dump( + { + "general_settings": {}, + "litellm_settings": {}, + "model_list": [], + "router_settings": {}, + }, + file, + ) + + print( + f"Config file not found. Created a default config file at {self.valves.LITELLM_CONFIG_DIR}" + ) + + with open(self.valves.LITELLM_CONFIG_DIR, "r") as file: + litellm_config = yaml.safe_load(file) + + self.valves.litellm_config = litellm_config + + asyncio.create_task(self.start_litellm_background()) + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + await self.shutdown_litellm_background() + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + + print(f"on_valves_updated:{__name__}") + + with open(self.valves.LITELLM_CONFIG_DIR, "r") as file: + litellm_config = yaml.safe_load(file) + + self.valves.litellm_config = litellm_config + + await self.shutdown_litellm_background() + await self.start_litellm_background() + pass + + async def run_background_process(self, command): + print("run_background_process") + + try: + # Log the command to be executed + print(f"Executing command: {command}") + + # Execute the command and create a subprocess + process = await asyncio.create_subprocess_exec( + *command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.background_process = process + print("Subprocess started successfully.") + + # Capture STDERR for debugging purposes + stderr_output = await process.stderr.read() + stderr_text = stderr_output.decode().strip() + if stderr_text: + print(f"Subprocess STDERR: {stderr_text}") + + # log.info output line by line + async for line in process.stdout: + print(line.decode().strip()) + + # Wait for the process to finish + returncode = await process.wait() + print(f"Subprocess exited with return code {returncode}") + except Exception as e: + print(f"Failed to start subprocess: {e}") + raise # Optionally re-raise the exception if you want it to propagate + + async def start_litellm_background(self): + print("start_litellm_background") + # Command to run in the background + command = [ + "litellm", + "--port", + str(self.valves.LITELLM_PROXY_PORT), + "--host", + self.valves.LITELLM_PROXY_HOST, + "--telemetry", + "False", + "--config", + self.valves.LITELLM_CONFIG_DIR, + ] + + await self.run_background_process(command) + + async def shutdown_litellm_background(self): + print("shutdown_litellm_background") + + if self.background_process: + self.background_process.terminate() + await self.background_process.wait() # Ensure the process has terminated + print("Subprocess terminated") + self.background_process = None + + def get_litellm_models(self): + if self.background_process: + try: + r = requests.get( + f"http://{self.valves.LITELLM_PROXY_HOST}:{self.valves.LITELLM_PROXY_PORT}/v1/models" + ) + models = r.json() + return [ + { + "id": model["id"], + "name": model["name"] if "name" in model else model["id"], + } + for model in models["data"] + ] + except Exception as e: + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from LiteLLM, please update the URL in the valves.", + }, + ] + else: + return [] + + # Pipelines are the models that are available in the manifold. + # It can be a list or a function that returns a list. + def pipelines(self) -> List[dict]: + return self.get_litellm_models() + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + if "user" in body: + print("######################################") + print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})') + print(f"# Message: {user_message}") + print("######################################") + + try: + r = requests.post( + url=f"http://{self.valves.LITELLM_PROXY_HOST}:{self.valves.LITELLM_PROXY_PORT}/v1/chat/completions", + json={**body, "model": model_id, "user": body["user"]["id"]}, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/llama_cpp_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/llama_cpp_pipeline.py new file mode 100644 index 0000000..51692da --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/llama_cpp_pipeline.py @@ -0,0 +1,61 @@ +""" +title: Llama C++ Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for generating responses using the Llama C++ library. +requirements: llama-cpp-python +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage + + +class Pipeline: + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "llama_cpp_pipeline" + + self.name = "Llama C++ Pipeline" + self.llm = None + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + from llama_cpp import Llama + + self.llm = Llama( + model_path="./models/llama3.gguf", + # n_gpu_layers=-1, # Uncomment to use GPU acceleration + # seed=1337, # Uncomment to set a specific seed + # n_ctx=2048, # Uncomment to increase the context window + ) + + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + print(body) + + response = self.llm.create_chat_completion_openai_v1( + messages=messages, + stream=body["stream"], + ) + + return response diff --git a/openwebui/pipelines/examples/pipelines/providers/mlx_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/mlx_manifold_pipeline.py new file mode 100644 index 0000000..2626090 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/mlx_manifold_pipeline.py @@ -0,0 +1,211 @@ +""" +title: MLX Manifold Pipeline +author: justinh-rahb +date: 2024-05-28 +version: 2.0 +license: MIT +description: A pipeline for generating text using Apple MLX Framework with dynamic model loading. +requirements: requests, mlx-lm, huggingface-hub, psutil +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import subprocess +import logging +from huggingface_hub import login +import time +import psutil + +class Pipeline: + class Valves(BaseModel): + MLX_DEFAULT_MODEL: str = "mlx-community/Meta-Llama-3-8B-Instruct-8bit" + MLX_MODEL_FILTER: str = "mlx-community" + MLX_STOP: str = "<|start_header_id|>,<|end_header_id|>,<|eot_id|>" + MLX_CHAT_TEMPLATE: str | None = None + MLX_USE_DEFAULT_CHAT_TEMPLATE: bool | None = False + HUGGINGFACE_TOKEN: str | None = None + + def __init__(self): + # Pipeline identification + self.type = "manifold" + self.id = "mlx" + self.name = "MLX/" + + # Initialize valves and update them + self.valves = self.Valves() + self.update_valves() + + # Server configuration + self.host = "localhost" # Always use localhost for security + self.port = None # Port will be dynamically assigned + + # Model management + self.models = self.get_mlx_models() + self.current_model = None + self.server_process = None + + # Start the MLX server with the default model + self.start_mlx_server(self.valves.MLX_DEFAULT_MODEL) + + def update_valves(self): + """Update pipeline configuration based on valve settings.""" + if self.valves.HUGGINGFACE_TOKEN: + login(self.valves.HUGGINGFACE_TOKEN) + self.stop_sequence = self.valves.MLX_STOP.split(",") + + def get_mlx_models(self): + """Fetch available MLX models based on the specified pattern.""" + try: + cmd = [ + 'mlx_lm.manage', + '--scan', + '--pattern', self.valves.MLX_MODEL_FILTER, + ] + result = subprocess.run(cmd, capture_output=True, text=True) + lines = result.stdout.strip().split('\n') + + content_lines = [line for line in lines if line and not line.startswith('-')] + + models = [] + for line in content_lines[2:]: # Skip header lines + parts = line.split() + if len(parts) >= 2: + repo_id = parts[0] + models.append({ + "id": f"{repo_id.split('/')[-1].lower()}", + "name": repo_id + }) + if not models: + # Add default model if no models are found + models.append({ + "id": f"mlx.{self.valves.MLX_DEFAULT_MODEL.split('/')[-1].lower()}", + "name": self.valves.MLX_DEFAULT_MODEL + }) + return models + except Exception as e: + logging.error(f"Error fetching MLX models: {e}") + # Return default model on error + return [{ + "id": f"mlx.{self.valves.MLX_DEFAULT_MODEL.split('/')[-1].lower()}", + "name": self.valves.MLX_DEFAULT_MODEL + }] + + def pipelines(self) -> List[dict]: + """Return the list of available models as pipelines.""" + return self.models + + def start_mlx_server(self, model_name): + """Start the MLX server with the specified model.""" + model_id = f"mlx.{model_name.split('/')[-1].lower()}" + if self.current_model == model_id and self.server_process and self.server_process.poll() is None: + logging.info(f"MLX server already running with model {model_name}") + return + + self.stop_mlx_server() + + self.port = self.find_free_port() + + command = [ + "mlx_lm.server", + "--model", model_name, + "--port", str(self.port), + ] + + # Add chat template options if specified + if self.valves.MLX_CHAT_TEMPLATE: + command.extend(["--chat-template", self.valves.MLX_CHAT_TEMPLATE]) + elif self.valves.MLX_USE_DEFAULT_CHAT_TEMPLATE: + command.append("--use-default-chat-template") + + logging.info(f"Starting MLX server with command: {' '.join(command)}") + self.server_process = subprocess.Popen(command) + self.current_model = model_id + logging.info(f"Started MLX server for model {model_name} on port {self.port}") + time.sleep(5) # Give the server some time to start up + + def stop_mlx_server(self): + """Stop the currently running MLX server.""" + if self.server_process: + try: + process = psutil.Process(self.server_process.pid) + for proc in process.children(recursive=True): + proc.terminate() + process.terminate() + process.wait(timeout=10) # Wait for the process to terminate + except psutil.NoSuchProcess: + pass # Process already terminated + except psutil.TimeoutExpired: + logging.warning("Timeout while terminating MLX server process") + finally: + self.server_process = None + self.current_model = None + self.port = None + logging.info("Stopped MLX server") + + def find_free_port(self): + """Find and return a free port to use for the MLX server.""" + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + return port + + async def on_startup(self): + """Perform any necessary startup operations.""" + logging.info(f"on_startup:{__name__}") + + async def on_shutdown(self): + """Perform cleanup operations on shutdown.""" + self.stop_mlx_server() + + async def on_valves_updated(self): + """Handle updates to the pipeline configuration.""" + self.update_valves() + self.models = self.get_mlx_models() + self.start_mlx_server(self.valves.MLX_DEFAULT_MODEL) + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + """Process a request through the MLX pipeline.""" + logging.info(f"pipe:{__name__}") + + # Switch model if necessary + if model_id != self.current_model: + model_name = next((model['name'] for model in self.models if model['id'] == model_id), self.valves.MLX_DEFAULT_MODEL) + self.start_mlx_server(model_name) + + url = f"http://{self.host}:{self.port}/v1/chat/completions" + headers = {"Content-Type": "application/json"} + + # Prepare the payload for the MLX server + max_tokens = body.get("max_tokens", 4096) + temperature = body.get("temperature", 0.8) + repeat_penalty = body.get("repeat_penalty", 1.0) + + payload = { + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + "repetition_penalty": repeat_penalty, + "stop": self.stop_sequence, + "stream": body.get("stream", False), + } + + try: + # Send request to MLX server + r = requests.post( + url, headers=headers, json=payload, stream=body.get("stream", False) + ) + r.raise_for_status() + + # Return streamed response or full JSON response + if body.get("stream", False): + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/mlx_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/mlx_pipeline.py new file mode 100644 index 0000000..8365ab2 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/mlx_pipeline.py @@ -0,0 +1,115 @@ +""" +title: MLX Pipeline +author: justinh-rahb +date: 2024-05-27 +version: 1.2 +license: MIT +description: A pipeline for generating text using Apple MLX Framework. +requirements: requests, mlx-lm, huggingface-hub +environment_variables: MLX_HOST, MLX_PORT, MLX_SUBPROCESS +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import requests +import os +import subprocess +import logging +from huggingface_hub import login + +class Pipeline: + class Valves(BaseModel): + MLX_MODEL: str = "mistralai/Mistral-7B-Instruct-v0.3" + MLX_STOP: str = "[INST]" + HUGGINGFACE_TOKEN: str = "" + + def __init__(self): + self.id = "mlx_pipeline" + self.name = "MLX Pipeline" + self.valves = self.Valves() + self.update_valves() + + self.host = os.getenv("MLX_HOST", "localhost") + self.port = os.getenv("MLX_PORT", "8080") + self.subprocess = os.getenv("MLX_SUBPROCESS", "true").lower() == "true" + + if self.subprocess: + self.start_mlx_server() + + def update_valves(self): + if self.valves.HUGGINGFACE_TOKEN: + login(self.valves.HUGGINGFACE_TOKEN) + self.stop_sequence = self.valves.MLX_STOP.split(",") + + def start_mlx_server(self): + if not os.getenv("MLX_PORT"): + self.port = self.find_free_port() + command = f"mlx_lm.server --model {self.valves.MLX_MODEL} --port {self.port}" + self.server_process = subprocess.Popen(command, shell=True) + logging.info(f"Started MLX server on port {self.port}") + + def find_free_port(self): + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + return port + + async def on_startup(self): + logging.info(f"on_startup:{__name__}") + + async def on_shutdown(self): + if self.subprocess and hasattr(self, "server_process"): + self.server_process.terminate() + logging.info(f"Terminated MLX server on port {self.port}") + + async def on_valves_updated(self): + self.update_valves() + if self.subprocess and hasattr(self, "server_process"): + self.server_process.terminate() + logging.info(f"Terminated MLX server on port {self.port}") + self.start_mlx_server() + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + logging.info(f"pipe:{__name__}") + + url = f"http://{self.host}:{self.port}/v1/chat/completions" + headers = {"Content-Type": "application/json"} + + max_tokens = body.get("max_tokens", 4096) + if not isinstance(max_tokens, int) or max_tokens < 0: + max_tokens = 4096 + + temperature = body.get("temperature", 0.8) + if not isinstance(temperature, (int, float)) or temperature < 0: + temperature = 0.8 + + repeat_penalty = body.get("repeat_penalty", 1.0) + if not isinstance(repeat_penalty, (int, float)) or repeat_penalty < 0: + repeat_penalty = 1.0 + + payload = { + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + "repetition_penalty": repeat_penalty, + "stop": self.stop_sequence, + "stream": body.get("stream", False), + } + + try: + r = requests.post( + url, headers=headers, json=payload, stream=body.get("stream", False) + ) + r.raise_for_status() + + if body.get("stream", False): + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" \ No newline at end of file diff --git a/openwebui/pipelines/examples/pipelines/providers/ollama_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/ollama_manifold_pipeline.py new file mode 100644 index 0000000..ec5c3b8 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/ollama_manifold_pipeline.py @@ -0,0 +1,99 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import os + +from pydantic import BaseModel +import requests + + +class Pipeline: + + class Valves(BaseModel): + OLLAMA_BASE_URL: str + + def __init__(self): + # You can also set the pipelines that are available in this pipeline. + # Set manifold to True if you want to use this pipeline as a manifold. + # Manifold pipelines can have multiple pipelines. + self.type = "manifold" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "ollama_manifold" + + # Optionally, you can set the name of the manifold pipeline. + self.name = "Ollama: " + + self.valves = self.Valves( + **{ + "OLLAMA_BASE_URL": os.getenv("OLLAMA_BASE_URL", "http://localhost:11435"), + } + ) + self.pipelines = [] + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + self.pipelines = self.get_ollama_models() + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + self.pipelines = self.get_ollama_models() + pass + + def get_ollama_models(self): + if self.valves.OLLAMA_BASE_URL: + try: + r = requests.get(f"{self.valves.OLLAMA_BASE_URL}/api/tags") + models = r.json() + return [ + {"id": model["model"], "name": model["name"]} + for model in models["models"] + ] + except Exception as e: + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from Ollama, please update the URL in the valves.", + }, + ] + else: + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + + if "user" in body: + print("######################################") + print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})') + print(f"# Message: {user_message}") + print("######################################") + + try: + r = requests.post( + url=f"{self.valves.OLLAMA_BASE_URL}/v1/chat/completions", + json={**body, "model": model_id}, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/ollama_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/ollama_pipeline.py new file mode 100644 index 0000000..d2560d6 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/ollama_pipeline.py @@ -0,0 +1,55 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import requests + + +class Pipeline: + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "ollama_pipeline" + self.name = "Ollama Pipeline" + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + OLLAMA_BASE_URL = "http://localhost:11434" + MODEL = "llama3" + + if "user" in body: + print("######################################") + print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})') + print(f"# Message: {user_message}") + print("######################################") + + try: + r = requests.post( + url=f"{OLLAMA_BASE_URL}/v1/chat/completions", + json={**body, "model": MODEL}, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/openai_dalle_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/openai_dalle_manifold_pipeline.py new file mode 100644 index 0000000..e35dbad --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/openai_dalle_manifold_pipeline.py @@ -0,0 +1,86 @@ +"""A manifold to integrate OpenAI's ImageGen models into Open-WebUI""" + +from typing import List, Union, Generator, Iterator + +from pydantic import BaseModel + +from openai import OpenAI + +class Pipeline: + """OpenAI ImageGen pipeline""" + + class Valves(BaseModel): + """Options to change from the WebUI""" + + OPENAI_API_BASE_URL: str = "https://api.openai.com/v1" + OPENAI_API_KEY: str = "" + IMAGE_SIZE: str = "1024x1024" + NUM_IMAGES: int = 1 + + def __init__(self): + self.type = "manifold" + self.name = "ImageGen: " + + self.valves = self.Valves() + self.client = OpenAI( + base_url=self.valves.OPENAI_API_BASE_URL, + api_key=self.valves.OPENAI_API_KEY, + ) + + self.pipelines = self.get_openai_assistants() + + async def on_startup(self) -> None: + """This function is called when the server is started.""" + print(f"on_startup:{__name__}") + + async def on_shutdown(self): + """This function is called when the server is stopped.""" + print(f"on_shutdown:{__name__}") + + async def on_valves_updated(self): + """This function is called when the valves are updated.""" + print(f"on_valves_updated:{__name__}") + self.client = OpenAI( + base_url=self.valves.OPENAI_API_BASE_URL, + api_key=self.valves.OPENAI_API_KEY, + ) + self.pipelines = self.get_openai_assistants() + + def get_openai_assistants(self) -> List[dict]: + """Get the available ImageGen models from OpenAI + + Returns: + List[dict]: The list of ImageGen models + """ + + if self.valves.OPENAI_API_KEY: + models = self.client.models.list() + return [ + { + "id": model.id, + "name": model.id, + } + for model in models + if "dall-e" in model.id + ] + + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + print(f"pipe:{__name__}") + + response = self.client.images.generate( + model=model_id, + prompt=user_message, + size=self.valves.IMAGE_SIZE, + n=self.valves.NUM_IMAGES, + ) + + message = "" + for image in response.data: + if image.url: + message += "![image](" + image.url + ")\n" + + yield message diff --git a/openwebui/pipelines/examples/pipelines/providers/openai_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/openai_manifold_pipeline.py new file mode 100644 index 0000000..84bc5cd --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/openai_manifold_pipeline.py @@ -0,0 +1,130 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel + +import os +import requests + + +class Pipeline: + class Valves(BaseModel): + OPENAI_API_BASE_URL: str = "https://api.openai.com/v1" + OPENAI_API_KEY: str = "" + pass + + def __init__(self): + self.type = "manifold" + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "openai_pipeline" + self.name = "OpenAI: " + + self.valves = self.Valves( + **{ + "OPENAI_API_KEY": os.getenv( + "OPENAI_API_KEY", "your-openai-api-key-here" + ) + } + ) + + self.pipelines = self.get_openai_models() + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + self.pipelines = self.get_openai_models() + pass + + def get_openai_models(self): + if self.valves.OPENAI_API_KEY: + try: + headers = {} + headers["Authorization"] = f"Bearer {self.valves.OPENAI_API_KEY}" + headers["Content-Type"] = "application/json" + + r = requests.get( + f"{self.valves.OPENAI_API_BASE_URL}/models", headers=headers + ) + + allowed_models = [ + "gpt", + "o1", + "o3", + "o4", + ] + + models = r.json() + return [ + { + "id": model["id"], + "name": model["name"] if "name" in model else model["id"], + } + for model in models["data"] + if any(substring in model["id"] for substring in allowed_models) + ] + + except Exception as e: + + print(f"Error: {e}") + return [ + { + "id": "error", + "name": "Could not fetch models from OpenAI, please update the API Key in the valves.", + }, + ] + else: + return [] + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + headers = {} + headers["Authorization"] = f"Bearer {self.valves.OPENAI_API_KEY}" + headers["Content-Type"] = "application/json" + + payload = {**body, "model": model_id} + + if "user" in payload: + del payload["user"] + if "chat_id" in payload: + del payload["chat_id"] + if "title" in payload: + del payload["title"] + + print(payload) + + try: + r = requests.post( + url=f"{self.valves.OPENAI_API_BASE_URL}/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/openai_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/openai_pipeline.py new file mode 100644 index 0000000..a7f97e5 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/openai_pipeline.py @@ -0,0 +1,81 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel +import os +import requests + + +class Pipeline: + class Valves(BaseModel): + OPENAI_API_KEY: str = "" + pass + + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "openai_pipeline" + self.name = "OpenAI Pipeline" + self.valves = self.Valves( + **{ + "OPENAI_API_KEY": os.getenv( + "OPENAI_API_KEY", "your-openai-api-key-here" + ) + } + ) + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + print(messages) + print(user_message) + + OPENAI_API_KEY = self.valves.OPENAI_API_KEY + MODEL = "gpt-3.5-turbo" + + headers = {} + headers["Authorization"] = f"Bearer {OPENAI_API_KEY}" + headers["Content-Type"] = "application/json" + + payload = {**body, "model": MODEL} + + if "user" in payload: + del payload["user"] + if "chat_id" in payload: + del payload["chat_id"] + if "title" in payload: + del payload["title"] + + print(payload) + + try: + r = requests.post( + url="https://api.openai.com/v1/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body["stream"]: + return r.iter_lines() + else: + return r.json() + except Exception as e: + return f"Error: {e}" diff --git a/openwebui/pipelines/examples/pipelines/providers/perplexity_manifold_pipeline.py b/openwebui/pipelines/examples/pipelines/providers/perplexity_manifold_pipeline.py new file mode 100644 index 0000000..c985b65 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/providers/perplexity_manifold_pipeline.py @@ -0,0 +1,167 @@ +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import os +import requests + +from utils.pipelines.main import pop_system_message + + +class Pipeline: + class Valves(BaseModel): + PERPLEXITY_API_BASE_URL: str = "https://api.perplexity.ai" + PERPLEXITY_API_KEY: str = "" + pass + + def __init__(self): + self.type = "manifold" + self.name = "Perplexity: " + + self.valves = self.Valves( + **{ + "PERPLEXITY_API_KEY": os.getenv( + "PERPLEXITY_API_KEY", "your-perplexity-api-key-here" + ) + } + ) + + # Debugging: print the API key to ensure it's loaded + print(f"Loaded API Key: {self.valves.PERPLEXITY_API_KEY}") + + # List of models + self.pipelines = [ + { + "id": "sonar-pro", + "name": "Sonar Pro" + }, + { + "id": "sonar", + "name": "Sonar" + }, + { + "id": "sonar-deep-research", + "name": "Sonar Deep Research" + }, + { + "id": "sonar-reasoning-pro", + "name": "Sonar Reasoning Pro" + }, + { + "id": "sonar-reasoning", "name": "Sonar Reasoning" + }, + { + "id": "r1-1776", "name": "R1-1776" + } + ] + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + print(f"on_valves_updated:{__name__}") + # No models to fetch, static setup + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + system_message, messages = pop_system_message(messages) + system_prompt = "You are a helpful assistant." + if system_message is not None: + system_prompt = system_message["content"] + + print(system_prompt) + print(messages) + print(user_message) + + headers = { + "Authorization": f"Bearer {self.valves.PERPLEXITY_API_KEY}", + "Content-Type": "application/json", + "accept": "application/json" + } + + payload = { + "model": model_id, + "messages": [ + {"role": "system", "content": system_prompt}, + *messages + ], + "stream": body.get("stream", True), + "return_citations": True, + "return_images": True + } + + if "user" in payload: + del payload["user"] + if "chat_id" in payload: + del payload["chat_id"] + if "title" in payload: + del payload["title"] + + print(payload) + + try: + r = requests.post( + url=f"{self.valves.PERPLEXITY_API_BASE_URL}/chat/completions", + json=payload, + headers=headers, + stream=True, + ) + + r.raise_for_status() + + if body.get("stream", False): + return r.iter_lines() + else: + response = r.json() + formatted_response = { + "id": response["id"], + "model": response["model"], + "created": response["created"], + "usage": response["usage"], + "object": response["object"], + "choices": [ + { + "index": choice["index"], + "finish_reason": choice["finish_reason"], + "message": { + "role": choice["message"]["role"], + "content": choice["message"]["content"] + }, + "delta": {"role": "assistant", "content": ""} + } for choice in response["choices"] + ] + } + return formatted_response + except Exception as e: + return f"Error: {e}" + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Perplexity API Client") + parser.add_argument("--api-key", type=str, required=True, + help="API key for Perplexity") + parser.add_argument("--prompt", type=str, required=True, + help="Prompt to send to the Perplexity API") + + args = parser.parse_args() + + pipeline = Pipeline() + pipeline.valves.PERPLEXITY_API_KEY = args.api_key + response = pipeline.pipe( + user_message=args.prompt, model_id="llama-3-sonar-large-32k-online", messages=[], body={"stream": False}) + + print("Response:", response) diff --git a/openwebui/pipelines/examples/pipelines/rag/haystack_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/haystack_pipeline.py new file mode 100644 index 0000000..41eb305 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/haystack_pipeline.py @@ -0,0 +1,108 @@ +""" +title: Haystack Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for retrieving relevant information from a knowledge base using the Haystack library. +requirements: haystack-ai, datasets>=2.6.1, sentence-transformers>=2.2.0 +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import os +import asyncio + + +class Pipeline: + def __init__(self): + self.basic_rag_pipeline = None + + async def on_startup(self): + os.environ["OPENAI_API_KEY"] = "your_openai_api_key_here" + + from haystack.components.embedders import SentenceTransformersDocumentEmbedder + from haystack.components.embedders import SentenceTransformersTextEmbedder + from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever + from haystack.components.builders import PromptBuilder + from haystack.components.generators import OpenAIGenerator + + from haystack.document_stores.in_memory import InMemoryDocumentStore + + from datasets import load_dataset + from haystack import Document + from haystack import Pipeline + + document_store = InMemoryDocumentStore() + + dataset = load_dataset("bilgeyucel/seven-wonders", split="train") + docs = [Document(content=doc["content"], meta=doc["meta"]) for doc in dataset] + + doc_embedder = SentenceTransformersDocumentEmbedder( + model="sentence-transformers/all-MiniLM-L6-v2" + ) + doc_embedder.warm_up() + + docs_with_embeddings = doc_embedder.run(docs) + document_store.write_documents(docs_with_embeddings["documents"]) + + text_embedder = SentenceTransformersTextEmbedder( + model="sentence-transformers/all-MiniLM-L6-v2" + ) + + retriever = InMemoryEmbeddingRetriever(document_store) + + template = """ + Given the following information, answer the question. + + Context: + {% for document in documents %} + {{ document.content }} + {% endfor %} + + Question: {{question}} + Answer: + """ + + prompt_builder = PromptBuilder(template=template) + + generator = OpenAIGenerator(model="gpt-3.5-turbo") + + self.basic_rag_pipeline = Pipeline() + # Add components to your pipeline + self.basic_rag_pipeline.add_component("text_embedder", text_embedder) + self.basic_rag_pipeline.add_component("retriever", retriever) + self.basic_rag_pipeline.add_component("prompt_builder", prompt_builder) + self.basic_rag_pipeline.add_component("llm", generator) + + # Now, connect the components to each other + self.basic_rag_pipeline.connect( + "text_embedder.embedding", "retriever.query_embedding" + ) + self.basic_rag_pipeline.connect("retriever", "prompt_builder.documents") + self.basic_rag_pipeline.connect("prompt_builder", "llm") + + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom RAG pipeline. + # Typically, you would retrieve relevant information from your knowledge base and synthesize it to generate a response. + + print(messages) + print(user_message) + + question = user_message + response = self.basic_rag_pipeline.run( + { + "text_embedder": {"text": question}, + "prompt_builder": {"question": question}, + } + ) + + return response["llm"]["replies"][0] diff --git a/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_github_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_github_pipeline.py new file mode 100644 index 0000000..41e4af8 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_github_pipeline.py @@ -0,0 +1,94 @@ +""" +title: Llama Index Ollama Github Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for retrieving relevant information from a knowledge base using the Llama Index library with Ollama embeddings from a GitHub repository. +requirements: llama-index, llama-index-llms-ollama, llama-index-embeddings-ollama, llama-index-readers-github +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import os +import asyncio + + +class Pipeline: + def __init__(self): + self.documents = None + self.index = None + + async def on_startup(self): + from llama_index.embeddings.ollama import OllamaEmbedding + from llama_index.llms.ollama import Ollama + from llama_index.core import VectorStoreIndex, Settings + from llama_index.readers.github import GithubRepositoryReader, GithubClient + + Settings.embed_model = OllamaEmbedding( + model_name="nomic-embed-text", + base_url="http://localhost:11434", + ) + Settings.llm = Ollama(model="llama3") + + global index, documents + + github_token = os.environ.get("GITHUB_TOKEN") + owner = "open-webui" + repo = "plugin-server" + branch = "main" + + github_client = GithubClient(github_token=github_token, verbose=True) + + reader = GithubRepositoryReader( + github_client=github_client, + owner=owner, + repo=repo, + use_parser=False, + verbose=False, + filter_file_extensions=( + [ + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + "json", + ".ipynb", + ], + GithubRepositoryReader.FilterType.EXCLUDE, + ), + ) + + loop = asyncio.new_event_loop() + + reader._loop = loop + + try: + # Load data from the branch + self.documents = await asyncio.to_thread(reader.load_data, branch=branch) + self.index = VectorStoreIndex.from_documents(self.documents) + finally: + loop.close() + + print(self.documents) + print(self.index) + + async def on_shutdown(self): + # This function is called when the server is stopped. + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom RAG pipeline. + # Typically, you would retrieve relevant information from your knowledge base and synthesize it to generate a response. + + print(messages) + print(user_message) + + query_engine = self.index.as_query_engine(streaming=True) + response = query_engine.query(user_message) + + return response.response_gen diff --git a/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_pipeline.py new file mode 100644 index 0000000..0b7765c --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/llamaindex_ollama_pipeline.py @@ -0,0 +1,74 @@ +""" +title: Llama Index Ollama Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for retrieving relevant information from a knowledge base using the Llama Index library with Ollama embeddings. +requirements: llama-index, llama-index-llms-ollama, llama-index-embeddings-ollama +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import os + +from pydantic import BaseModel + + +class Pipeline: + + class Valves(BaseModel): + LLAMAINDEX_OLLAMA_BASE_URL: str + LLAMAINDEX_MODEL_NAME: str + LLAMAINDEX_EMBEDDING_MODEL_NAME: str + + def __init__(self): + self.documents = None + self.index = None + + self.valves = self.Valves( + **{ + "LLAMAINDEX_OLLAMA_BASE_URL": os.getenv("LLAMAINDEX_OLLAMA_BASE_URL", "http://localhost:11434"), + "LLAMAINDEX_MODEL_NAME": os.getenv("LLAMAINDEX_MODEL_NAME", "llama3"), + "LLAMAINDEX_EMBEDDING_MODEL_NAME": os.getenv("LLAMAINDEX_EMBEDDING_MODEL_NAME", "nomic-embed-text"), + } + ) + + async def on_startup(self): + from llama_index.embeddings.ollama import OllamaEmbedding + from llama_index.llms.ollama import Ollama + from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader + + Settings.embed_model = OllamaEmbedding( + model_name=self.valves.LLAMAINDEX_EMBEDDING_MODEL_NAME, + base_url=self.valves.LLAMAINDEX_OLLAMA_BASE_URL, + ) + Settings.llm = Ollama( + model=self.valves.LLAMAINDEX_MODEL_NAME, + base_url=self.valves.LLAMAINDEX_OLLAMA_BASE_URL, + ) + + # This function is called when the server is started. + global documents, index + + self.documents = SimpleDirectoryReader("/app/backend/data").load_data() + self.index = VectorStoreIndex.from_documents(self.documents) + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom RAG pipeline. + # Typically, you would retrieve relevant information from your knowledge base and synthesize it to generate a response. + + print(messages) + print(user_message) + + query_engine = self.index.as_query_engine(streaming=True) + response = query_engine.query(user_message) + + return response.response_gen diff --git a/openwebui/pipelines/examples/pipelines/rag/llamaindex_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/llamaindex_pipeline.py new file mode 100644 index 0000000..2606361 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/llamaindex_pipeline.py @@ -0,0 +1,49 @@ +""" +title: Llama Index Pipeline +author: open-webui +date: 2024-05-30 +version: 1.0 +license: MIT +description: A pipeline for retrieving relevant information from a knowledge base using the Llama Index library. +requirements: llama-index +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage + + +class Pipeline: + def __init__(self): + self.documents = None + self.index = None + + async def on_startup(self): + import os + + # Set the OpenAI API key + os.environ["OPENAI_API_KEY"] = "your-api-key-here" + + from llama_index.core import VectorStoreIndex, SimpleDirectoryReader + + self.documents = SimpleDirectoryReader("./data").load_data() + self.index = VectorStoreIndex.from_documents(self.documents) + # This function is called when the server is started. + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom RAG pipeline. + # Typically, you would retrieve relevant information from your knowledge base and synthesize it to generate a response. + + print(messages) + print(user_message) + + query_engine = self.index.as_query_engine(streaming=True) + response = query_engine.query(user_message) + + return response.response_gen diff --git a/openwebui/pipelines/examples/pipelines/rag/r2r_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/r2r_pipeline.py new file mode 100644 index 0000000..1dbcc15 --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/r2r_pipeline.py @@ -0,0 +1,45 @@ +""" +title: R2R Pipeline +author: Nolan Tremelling +date: 2025-03-21 +version: 1.0 +license: MIT +description: A pipeline for retrieving relevant information from a knowledge base using R2R. +requirements: r2r +""" + +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +import os +import asyncio + + +class Pipeline: + def __init__(self): + self.r2r_client = None + + async def on_startup(self): + from r2r import R2RClient + + # Connect to either SciPhi cloud or your self hosted R2R server + self.r2r_client = R2RClient(os.getenv("R2R_SERVER_URL", "https://api.sciphi.ai")) + self.r2r_client.set_api_key(os.getenv("R2R_API_KEY", "")) + + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + self.r2r_client = None + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + + print(messages) + print(user_message) + + response = self.r2r_client.retrieval.rag( + query=user_message, + ) + + return response.results.completion diff --git a/openwebui/pipelines/examples/pipelines/rag/text_to_sql_pipeline.py b/openwebui/pipelines/examples/pipelines/rag/text_to_sql_pipeline.py new file mode 100644 index 0000000..31471ad --- /dev/null +++ b/openwebui/pipelines/examples/pipelines/rag/text_to_sql_pipeline.py @@ -0,0 +1,111 @@ +""" +title: Llama Index DB Pipeline +author: 0xThresh +date: 2024-08-11 +version: 1.1 +license: MIT +description: A pipeline for using text-to-SQL for retrieving relevant information from a database using the Llama Index library. +requirements: llama_index, sqlalchemy, psycopg2-binary +""" + +from typing import List, Union, Generator, Iterator +import os +from pydantic import BaseModel +from llama_index.llms.ollama import Ollama +from llama_index.core.query_engine import NLSQLTableQueryEngine +from llama_index.core import SQLDatabase, PromptTemplate +from sqlalchemy import create_engine + + +class Pipeline: + class Valves(BaseModel): + DB_HOST: str + DB_PORT: str + DB_USER: str + DB_PASSWORD: str + DB_DATABASE: str + DB_TABLE: str + OLLAMA_HOST: str + TEXT_TO_SQL_MODEL: str + + + # Update valves/ environment variables based on your selected database + def __init__(self): + self.name = "Database RAG Pipeline" + self.engine = None + self.nlsql_response = "" + + # Initialize + self.valves = self.Valves( + **{ + "pipelines": ["*"], # Connect to all pipelines + "DB_HOST": os.getenv("DB_HOST", "http://localhost"), # Database hostname + "DB_PORT": os.getenv("DB_PORT", 5432), # Database port + "DB_USER": os.getenv("DB_USER", "postgres"), # User to connect to the database with + "DB_PASSWORD": os.getenv("DB_PASSWORD", "password"), # Password to connect to the database with + "DB_DATABASE": os.getenv("DB_DATABASE", "postgres"), # Database to select on the DB instance + "DB_TABLE": os.getenv("DB_TABLE", "table_name"), # Table(s) to run queries against + "OLLAMA_HOST": os.getenv("OLLAMA_HOST", "http://host.docker.internal:11434"), # Make sure to update with the URL of your Ollama host, such as http://localhost:11434 or remote server address + "TEXT_TO_SQL_MODEL": os.getenv("TEXT_TO_SQL_MODEL", "llama3.1:latest") # Model to use for text-to-SQL generation + } + ) + + def init_db_connection(self): + # Update your DB connection string based on selected DB engine - current connection string is for Postgres + self.engine = create_engine(f"postgresql+psycopg2://{self.valves.DB_USER}:{self.valves.DB_PASSWORD}@{self.valves.DB_HOST}:{self.valves.DB_PORT}/{self.valves.DB_DATABASE}") + return self.engine + + async def on_startup(self): + # This function is called when the server is started. + self.init_db_connection() + + async def on_shutdown(self): + # This function is called when the server is stopped. + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # Debug logging is required to see what SQL query is generated by the LlamaIndex library; enable on Pipelines server if needed + + # Create database reader for Postgres + sql_database = SQLDatabase(self.engine, include_tables=[self.valves.DB_TABLE]) + + # Set up LLM connection; uses phi3 model with 128k context limit since some queries have returned 20k+ tokens + llm = Ollama(model=self.valves.TEXT_TO_SQL_MODEL, base_url=self.valves.OLLAMA_HOST, request_timeout=180.0, context_window=30000) + + # Set up the custom prompt used when generating SQL queries from text + text_to_sql_prompt = """ + Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. + You can order the results by a relevant column to return the most interesting examples in the database. + Unless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per Postgres. You can order the results to return the most informative data in the database. + Never query for all the columns from a specific table, only ask for a few relevant columns given the question. + You should use DISTINCT statements and avoid returning duplicates wherever possible. + Pay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist. Pay attention to which column is in which table. Also, qualify column names with the table name when needed. You are required to use the following format, each taking one line: + + Question: Question here + SQLQuery: SQL Query to run + SQLResult: Result of the SQLQuery + Answer: Final answer here + + Only use tables listed below. + {schema} + + Question: {query_str} + SQLQuery: + """ + + text_to_sql_template = PromptTemplate(text_to_sql_prompt) + + query_engine = NLSQLTableQueryEngine( + sql_database=sql_database, + tables=[self.valves.DB_TABLE], + llm=llm, + embed_model="local", + text_to_sql_prompt=text_to_sql_template, + streaming=True + ) + + response = query_engine.query(user_message) + + return response.response_gen diff --git a/openwebui/pipelines/examples/scaffolds/example_pipeline_scaffold.py b/openwebui/pipelines/examples/scaffolds/example_pipeline_scaffold.py new file mode 100644 index 0000000..cb0ec11 --- /dev/null +++ b/openwebui/pipelines/examples/scaffolds/example_pipeline_scaffold.py @@ -0,0 +1,67 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage +from pydantic import BaseModel + + +class Pipeline: + class Valves(BaseModel): + pass + + def __init__(self): + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "pipeline_example" + + # The name of the pipeline. + self.name = "Pipeline Example" + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def on_valves_updated(self): + # This function is called when the valves are updated. + pass + + async def inlet(self, body: dict, user: dict) -> dict: + # This function is called before the OpenAI API request is made. You can modify the form data before it is sent to the OpenAI API. + print(f"inlet:{__name__}") + + print(body) + print(user) + + return body + + async def outlet(self, body: dict, user: dict) -> dict: + # This function is called after the OpenAI API response is completed. You can modify the messages after they are received from the OpenAI API. + print(f"outlet:{__name__}") + + print(body) + print(user) + + return body + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + # If you'd like to check for title generation, you can add the following check + if body.get("title", False): + print("Title Generation Request") + + print(messages) + print(user_message) + print(body) + + return f"{__name__} response to: {user_message}" diff --git a/openwebui/pipelines/examples/scaffolds/filter_pipeline_scaffold.py b/openwebui/pipelines/examples/scaffolds/filter_pipeline_scaffold.py new file mode 100644 index 0000000..cc08434 --- /dev/null +++ b/openwebui/pipelines/examples/scaffolds/filter_pipeline_scaffold.py @@ -0,0 +1,68 @@ +""" +title: Filter Pipeline +author: open-webui +date: 2024-05-30 +version: 1.1 +license: MIT +description: Example of a filter pipeline that can be used to edit the form data before it is sent to the OpenAI API. +requirements: requests +""" + +from typing import List, Optional +from pydantic import BaseModel +from schemas import OpenAIChatMessage + + +class Pipeline: + class Valves(BaseModel): + # List target pipeline ids (models) that this filter will be connected to. + # If you want to connect this filter to all pipelines, you can set pipelines to ["*"] + pipelines: List[str] = [] + + # Assign a priority level to the filter pipeline. + # The priority level determines the order in which the filter pipelines are executed. + # The lower the number, the higher the priority. + priority: int = 0 + + # Add your custom parameters here + pass + + def __init__(self): + # Pipeline filters are only compatible with Open WebUI + # You can think of filter pipeline as a middleware that can be used to edit the form data before it is sent to the OpenAI API. + self.type = "filter" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "filter_pipeline" + + self.name = "Filter" + + self.valves = self.Valves(**{"pipelines": ["llama3:latest"]}) + + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + async def inlet(self, body: dict, user: Optional[dict] = None) -> dict: + # This filter is applied to the form data before it is sent to the OpenAI API. + print(f"inlet:{__name__}") + + # If you'd like to check for title generation, you can add the following check + if body.get("title", False): + print("Title Generation Request") + + print(body) + print(user) + + return body diff --git a/openwebui/pipelines/examples/scaffolds/function_calling_scaffold.py b/openwebui/pipelines/examples/scaffolds/function_calling_scaffold.py new file mode 100644 index 0000000..63940e6 --- /dev/null +++ b/openwebui/pipelines/examples/scaffolds/function_calling_scaffold.py @@ -0,0 +1,33 @@ +from blueprints.function_calling_blueprint import Pipeline as FunctionCallingBlueprint + + +class Pipeline(FunctionCallingBlueprint): + class Valves(FunctionCallingBlueprint.Valves): + # Add your custom valves here + pass + + class Tools: + def __init__(self, pipeline) -> None: + self.pipeline = pipeline + + # Add your custom tools using pure Python code here, make sure to add type hints + # Use Sphinx-style docstrings to document your tools, they will be used for generating tools specifications + # Please refer to function_calling_filter_pipeline.py for an example + pass + + def __init__(self): + super().__init__() + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "my_tools_pipeline" + self.name = "My Tools Pipeline" + self.valves = self.Valves( + **{ + **self.valves.model_dump(), + "pipelines": ["*"], # Connect to all pipelines + }, + ) + self.tools = self.Tools(self) diff --git a/openwebui/pipelines/examples/scaffolds/manifold_pipeline_scaffold.py b/openwebui/pipelines/examples/scaffolds/manifold_pipeline_scaffold.py new file mode 100644 index 0000000..eaff91e --- /dev/null +++ b/openwebui/pipelines/examples/scaffolds/manifold_pipeline_scaffold.py @@ -0,0 +1,59 @@ +from typing import List, Union, Generator, Iterator +from schemas import OpenAIChatMessage + + +class Pipeline: + def __init__(self): + # You can also set the pipelines that are available in this pipeline. + # Set manifold to True if you want to use this pipeline as a manifold. + # Manifold pipelines can have multiple pipelines. + self.type = "manifold" + + # Optionally, you can set the id and name of the pipeline. + # Best practice is to not specify the id so that it can be automatically inferred from the filename, so that users can install multiple versions of the same pipeline. + # The identifier must be unique across all pipelines. + # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes. + # self.id = "manifold_pipeline" + + # Optionally, you can set the name of the manifold pipeline. + self.name = "Manifold: " + + # Define pipelines that are available in this manifold pipeline. + # This is a list of dictionaries where each dictionary has an id and name. + self.pipelines = [ + { + "id": "pipeline-1", # This will turn into `manifold_pipeline.pipeline-1` + "name": "Pipeline 1", # This will turn into `Manifold: Pipeline 1` + }, + { + "id": "pipeline-2", + "name": "Pipeline 2", + }, + ] + pass + + async def on_startup(self): + # This function is called when the server is started. + print(f"on_startup:{__name__}") + pass + + async def on_shutdown(self): + # This function is called when the server is stopped. + print(f"on_shutdown:{__name__}") + pass + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + # This is where you can add your custom pipelines like RAG. + print(f"pipe:{__name__}") + + # If you'd like to check for title generation, you can add the following check + if body.get("title", False): + print("Title Generation Request") + + print(messages) + print(user_message) + print(body) + + return f"{model_id} response to: {user_message}" diff --git a/openwebui/pipelines/main.py b/openwebui/pipelines/main.py new file mode 100644 index 0000000..e277d3a --- /dev/null +++ b/openwebui/pipelines/main.py @@ -0,0 +1,789 @@ +from fastapi import FastAPI, Request, Depends, status, HTTPException, UploadFile, File +from fastapi.middleware.cors import CORSMiddleware +from fastapi.concurrency import run_in_threadpool + + +from starlette.responses import StreamingResponse, Response +from pydantic import BaseModel, ConfigDict +from typing import List, Union, Generator, Iterator + + +from utils.pipelines.auth import bearer_security, get_current_user +from utils.pipelines.main import get_last_user_message, stream_message_template +from utils.pipelines.misc import convert_to_raw_url + +from contextlib import asynccontextmanager +from concurrent.futures import ThreadPoolExecutor +from schemas import FilterForm, OpenAIChatCompletionForm +from urllib.parse import urlparse + +import shutil +import aiohttp +import os +import importlib.util +import logging +import time +import json +import uuid +import sys +import subprocess + + +from config import API_KEY, PIPELINES_DIR, LOG_LEVELS + +if not os.path.exists(PIPELINES_DIR): + os.makedirs(PIPELINES_DIR) + + +PIPELINES = {} +PIPELINE_MODULES = {} +PIPELINE_NAMES = {} + +# Add GLOBAL_LOG_LEVEL for Pipeplines +log_level = os.getenv("GLOBAL_LOG_LEVEL", "INFO").upper() +logging.basicConfig(level=LOG_LEVELS[log_level]) + + +def get_all_pipelines(): + pipelines = {} + for pipeline_id in PIPELINE_MODULES.keys(): + pipeline = PIPELINE_MODULES[pipeline_id] + + if hasattr(pipeline, "type"): + if pipeline.type == "manifold": + manifold_pipelines = [] + + # Check if pipelines is a function or a list + if callable(pipeline.pipelines): + manifold_pipelines = pipeline.pipelines() + else: + manifold_pipelines = pipeline.pipelines + + for p in manifold_pipelines: + manifold_pipeline_id = f'{pipeline_id}.{p["id"]}' + + manifold_pipeline_name = p["name"] + if hasattr(pipeline, "name"): + manifold_pipeline_name = ( + f"{pipeline.name}{manifold_pipeline_name}" + ) + + pipelines[manifold_pipeline_id] = { + "module": pipeline_id, + "type": pipeline.type if hasattr(pipeline, "type") else "pipe", + "id": manifold_pipeline_id, + "name": manifold_pipeline_name, + "valves": ( + pipeline.valves if hasattr(pipeline, "valves") else None + ), + } + if pipeline.type == "filter": + pipelines[pipeline_id] = { + "module": pipeline_id, + "type": (pipeline.type if hasattr(pipeline, "type") else "pipe"), + "id": pipeline_id, + "name": ( + pipeline.name if hasattr(pipeline, "name") else pipeline_id + ), + "pipelines": ( + pipeline.valves.pipelines + if hasattr(pipeline, "valves") + and hasattr(pipeline.valves, "pipelines") + else [] + ), + "priority": ( + pipeline.valves.priority + if hasattr(pipeline, "valves") + and hasattr(pipeline.valves, "priority") + else 0 + ), + "valves": pipeline.valves if hasattr(pipeline, "valves") else None, + } + else: + pipelines[pipeline_id] = { + "module": pipeline_id, + "type": (pipeline.type if hasattr(pipeline, "type") else "pipe"), + "id": pipeline_id, + "name": (pipeline.name if hasattr(pipeline, "name") else pipeline_id), + "valves": pipeline.valves if hasattr(pipeline, "valves") else None, + } + + return pipelines + + +def parse_frontmatter(content): + frontmatter = {} + for line in content.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + frontmatter[key.strip().lower()] = value.strip() + return frontmatter + + +def install_frontmatter_requirements(requirements): + if requirements: + req_list = [req.strip() for req in requirements.split(",")] + for req in req_list: + print(f"Installing requirement: {req}") + subprocess.check_call([sys.executable, "-m", "pip", "install", req]) + else: + print("No requirements found in frontmatter.") + + +async def load_module_from_path(module_name, module_path): + + try: + # Read the module content + with open(module_path, "r") as file: + content = file.read() + + # Parse frontmatter + frontmatter = {} + if content.startswith('"""'): + end = content.find('"""', 3) + if end != -1: + frontmatter_content = content[3:end] + frontmatter = parse_frontmatter(frontmatter_content) + + # Install requirements if specified + if "requirements" in frontmatter: + install_frontmatter_requirements(frontmatter["requirements"]) + + # Load the module + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + print(f"Loaded module: {module.__name__}") + if hasattr(module, "Pipeline"): + return module.Pipeline() + else: + raise Exception("No Pipeline class found") + except Exception as e: + print(f"Error loading module: {module_name}") + + # Move the file to the error folder + failed_pipelines_folder = os.path.join(PIPELINES_DIR, "failed") + if not os.path.exists(failed_pipelines_folder): + os.makedirs(failed_pipelines_folder) + + failed_file_path = os.path.join(failed_pipelines_folder, f"{module_name}.py") + os.rename(module_path, failed_file_path) + print(e) + return None + + +async def load_modules_from_directory(directory): + global PIPELINE_MODULES + global PIPELINE_NAMES + + for filename in os.listdir(directory): + if filename.endswith(".py"): + module_name = filename[:-3] # Remove the .py extension + module_path = os.path.join(directory, filename) + + # Create subfolder matching the filename without the .py extension + subfolder_path = os.path.join(directory, module_name) + if not os.path.exists(subfolder_path): + os.makedirs(subfolder_path) + logging.info(f"Created subfolder: {subfolder_path}") + + # Create a valves.json file if it doesn't exist + valves_json_path = os.path.join(subfolder_path, "valves.json") + if not os.path.exists(valves_json_path): + with open(valves_json_path, "w") as f: + json.dump({}, f) + logging.info(f"Created valves.json in: {subfolder_path}") + + pipeline = await load_module_from_path(module_name, module_path) + if pipeline: + # Overwrite pipeline.valves with values from valves.json + if os.path.exists(valves_json_path): + with open(valves_json_path, "r") as f: + valves_json = json.load(f) + if hasattr(pipeline, "valves"): + ValvesModel = pipeline.valves.__class__ + # Create a ValvesModel instance using default values and overwrite with valves_json + combined_valves = { + **pipeline.valves.model_dump(), + **valves_json, + } + valves = ValvesModel(**combined_valves) + pipeline.valves = valves + + logging.info(f"Updated valves for module: {module_name}") + + pipeline_id = pipeline.id if hasattr(pipeline, "id") else module_name + PIPELINE_MODULES[pipeline_id] = pipeline + PIPELINE_NAMES[pipeline_id] = module_name + logging.info(f"Loaded module: {module_name}") + else: + logging.warning(f"No Pipeline class found in {module_name}") + + global PIPELINES + PIPELINES = get_all_pipelines() + + +async def on_startup(): + await load_modules_from_directory(PIPELINES_DIR) + + for module in PIPELINE_MODULES.values(): + if hasattr(module, "on_startup"): + await module.on_startup() + + +async def on_shutdown(): + for module in PIPELINE_MODULES.values(): + if hasattr(module, "on_shutdown"): + await module.on_shutdown() + + +async def reload(): + await on_shutdown() + # Clear existing pipelines + PIPELINES.clear() + PIPELINE_MODULES.clear() + PIPELINE_NAMES.clear() + # Load pipelines afresh + await on_startup() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await on_startup() + yield + await on_shutdown() + + +app = FastAPI(docs_url="/docs", redoc_url=None, lifespan=lifespan) + +app.state.PIPELINES = PIPELINES + + +origins = ["*"] + + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.middleware("http") +async def check_url(request: Request, call_next): + start_time = int(time.time()) + app.state.PIPELINES = get_all_pipelines() + response = await call_next(request) + process_time = int(time.time()) - start_time + response.headers["X-Process-Time"] = str(process_time) + + return response + + +@app.get("/v1/models") +@app.get("/models") +async def get_models(user: str = Depends(get_current_user)): + """ + Returns the available pipelines + """ + app.state.PIPELINES = get_all_pipelines() + return { + "data": [ + { + "id": pipeline["id"], + "name": pipeline["name"], + "object": "model", + "created": int(time.time()), + "owned_by": "openai", + "pipeline": { + "type": pipeline["type"], + **( + { + "pipelines": ( + pipeline["valves"].pipelines + if pipeline.get("valves", None) + else [] + ), + "priority": pipeline.get("priority", 0), + } + if pipeline.get("type", "pipe") == "filter" + else {} + ), + "valves": pipeline["valves"] != None, + }, + } + for pipeline in app.state.PIPELINES.values() + ], + "object": "list", + "pipelines": True, + } + + +@app.get("/v1") +@app.get("/") +async def get_status(): + return {"status": True} + + +@app.get("/v1/pipelines") +@app.get("/pipelines") +async def list_pipelines(user: str = Depends(get_current_user)): + if user == API_KEY: + return { + "data": [ + { + "id": pipeline_id, + "name": PIPELINE_NAMES[pipeline_id], + "type": ( + PIPELINE_MODULES[pipeline_id].type + if hasattr(PIPELINE_MODULES[pipeline_id], "type") + else "pipe" + ), + "valves": ( + True + if hasattr(PIPELINE_MODULES[pipeline_id], "valves") + else False + ), + } + for pipeline_id in list(PIPELINE_MODULES.keys()) + ] + } + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + +class AddPipelineForm(BaseModel): + url: str + + +async def download_file(url: str, dest_folder: str): + filename = os.path.basename(urlparse(url).path) + if not filename.endswith(".py"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="URL must point to a Python file", + ) + + file_path = os.path.join(dest_folder, filename) + + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status != 200: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Failed to download file", + ) + with open(file_path, "wb") as f: + f.write(await response.read()) + + return file_path + + +@app.post("/v1/pipelines/add") +@app.post("/pipelines/add") +async def add_pipeline( + form_data: AddPipelineForm, user: str = Depends(get_current_user) +): + if user != API_KEY: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + try: + url = convert_to_raw_url(form_data.url) + + print(url) + file_path = await download_file(url, dest_folder=PIPELINES_DIR) + await reload() + return { + "status": True, + "detail": f"Pipeline added successfully from {file_path}", + } + except HTTPException as e: + raise e + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e), + ) + + +@app.post("/v1/pipelines/upload") +@app.post("/pipelines/upload") +async def upload_pipeline( + file: UploadFile = File(...), user: str = Depends(get_current_user) +): + if user != API_KEY: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + file_ext = os.path.splitext(file.filename)[1] + if file_ext != ".py": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only Python files are allowed.", + ) + + try: + # Ensure the destination folder exists + os.makedirs(PIPELINES_DIR, exist_ok=True) + + # Define the file path + file_path = os.path.join(PIPELINES_DIR, file.filename) + + # Save the uploaded file to the specified directory + with open(file_path, "wb") as buffer: + shutil.copyfileobj(file.file, buffer) + + # Perform any necessary reload or processing + await reload() + + return { + "status": True, + "detail": f"Pipeline uploaded successfully to {file_path}", + } + except HTTPException as e: + raise e + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e), + ) + + +class DeletePipelineForm(BaseModel): + id: str + + +@app.delete("/v1/pipelines/delete") +@app.delete("/pipelines/delete") +async def delete_pipeline( + form_data: DeletePipelineForm, user: str = Depends(get_current_user) +): + if user != API_KEY: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + pipeline_id = form_data.id + pipeline_name = PIPELINE_NAMES.get(pipeline_id.split(".")[0], None) + + if PIPELINE_MODULES[pipeline_id]: + if hasattr(PIPELINE_MODULES[pipeline_id], "on_shutdown"): + await PIPELINE_MODULES[pipeline_id].on_shutdown() + + pipeline_path = os.path.join(PIPELINES_DIR, f"{pipeline_name}.py") + if os.path.exists(pipeline_path): + os.remove(pipeline_path) + await reload() + return { + "status": True, + "detail": f"Pipeline {pipeline_id} deleted successfully", + } + else: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Pipeline {pipeline_id} not found", + ) + + +@app.post("/v1/pipelines/reload") +@app.post("/pipelines/reload") +async def reload_pipelines(user: str = Depends(get_current_user)): + if user == API_KEY: + await reload() + return {"message": "Pipelines reloaded successfully."} + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + +@app.get("/v1/{pipeline_id}/valves") +@app.get("/{pipeline_id}/valves") +async def get_valves(pipeline_id: str): + if pipeline_id not in PIPELINE_MODULES: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Pipeline {pipeline_id} not found", + ) + + pipeline = PIPELINE_MODULES[pipeline_id] + + if hasattr(pipeline, "valves") is False: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Valves for {pipeline_id} not found", + ) + + return pipeline.valves + + +@app.get("/v1/{pipeline_id}/valves/spec") +@app.get("/{pipeline_id}/valves/spec") +async def get_valves_spec(pipeline_id: str): + if pipeline_id not in PIPELINE_MODULES: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Pipeline {pipeline_id} not found", + ) + + pipeline = PIPELINE_MODULES[pipeline_id] + + if hasattr(pipeline, "valves") is False: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Valves for {pipeline_id} not found", + ) + + return pipeline.valves.schema() + + +@app.post("/v1/{pipeline_id}/valves/update") +@app.post("/{pipeline_id}/valves/update") +async def update_valves(pipeline_id: str, form_data: dict): + + if pipeline_id not in PIPELINE_MODULES: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Pipeline {pipeline_id} not found", + ) + + pipeline = PIPELINE_MODULES[pipeline_id] + + if hasattr(pipeline, "valves") is False: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Valves for {pipeline_id} not found", + ) + + try: + ValvesModel = pipeline.valves.__class__ + valves = ValvesModel(**form_data) + pipeline.valves = valves + + # Determine the directory path for the valves.json file + subfolder_path = os.path.join(PIPELINES_DIR, PIPELINE_NAMES[pipeline_id]) + valves_json_path = os.path.join(subfolder_path, "valves.json") + + # Save the updated valves data back to the valves.json file + with open(valves_json_path, "w") as f: + json.dump(valves.model_dump(), f) + + if hasattr(pipeline, "on_valves_updated"): + await pipeline.on_valves_updated() + except Exception as e: + print(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"{str(e)}", + ) + + return pipeline.valves + + +@app.post("/v1/{pipeline_id}/filter/inlet") +@app.post("/{pipeline_id}/filter/inlet") +async def filter_inlet(pipeline_id: str, form_data: FilterForm): + if pipeline_id not in app.state.PIPELINES: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Filter {pipeline_id} not found", + ) + + try: + pipeline = app.state.PIPELINES[form_data.body["model"]] + if pipeline["type"] == "manifold": + pipeline_id = pipeline_id.split(".")[0] + except: + pass + + pipeline = PIPELINE_MODULES[pipeline_id] + + try: + if hasattr(pipeline, "inlet"): + body = await pipeline.inlet(form_data.body, form_data.user) + return body + else: + return form_data.body + except Exception as e: + print(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"{str(e)}", + ) + + +@app.post("/v1/{pipeline_id}/filter/outlet") +@app.post("/{pipeline_id}/filter/outlet") +async def filter_outlet(pipeline_id: str, form_data: FilterForm): + if pipeline_id not in app.state.PIPELINES: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Filter {pipeline_id} not found", + ) + + try: + pipeline = app.state.PIPELINES[form_data.body["model"]] + if pipeline["type"] == "manifold": + pipeline_id = pipeline_id.split(".")[0] + except: + pass + + pipeline = PIPELINE_MODULES[pipeline_id] + + try: + if hasattr(pipeline, "outlet"): + body = await pipeline.outlet(form_data.body, form_data.user) + return body + else: + return form_data.body + except Exception as e: + print(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"{str(e)}", + ) + + +@app.post("/v1/chat/completions") +@app.post("/chat/completions") +async def generate_openai_chat_completion(form_data: OpenAIChatCompletionForm): + messages = [message.model_dump() for message in form_data.messages] + user_message = get_last_user_message(messages) + + if ( + form_data.model not in app.state.PIPELINES + or app.state.PIPELINES[form_data.model]["type"] == "filter" + ): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Pipeline {form_data.model} not found", + ) + + def job(): + print(form_data.model) + + pipeline = app.state.PIPELINES[form_data.model] + pipeline_id = form_data.model + + print(pipeline_id) + + if pipeline["type"] == "manifold": + manifold_id, pipeline_id = pipeline_id.split(".", 1) + pipe = PIPELINE_MODULES[manifold_id].pipe + else: + pipe = PIPELINE_MODULES[pipeline_id].pipe + + if form_data.stream: + + def stream_content(): + res = pipe( + user_message=user_message, + model_id=pipeline_id, + messages=messages, + body=form_data.model_dump(), + ) + logging.info(f"stream:true:{res}") + + if isinstance(res, str): + message = stream_message_template(form_data.model, res) + logging.info(f"stream_content:str:{message}") + yield f"data: {json.dumps(message)}\n\n" + + if isinstance(res, Iterator): + for line in res: + if isinstance(line, BaseModel): + line = line.model_dump_json() + line = f"data: {line}" + + elif isinstance(line, dict): + line = json.dumps(line) + line = f"data: {line}" + + try: + line = line.decode("utf-8") + logging.info(f"stream_content:Generator:{line}") + except: + pass + + if isinstance(line, str) and line.startswith("data:"): + yield f"{line}\n\n" + else: + line = stream_message_template(form_data.model, line) + yield f"data: {json.dumps(line)}\n\n" + + if isinstance(res, str) or isinstance(res, Generator): + finish_message = { + "id": f"{form_data.model}-{str(uuid.uuid4())}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": form_data.model, + "choices": [ + { + "index": 0, + "delta": {}, + "logprobs": None, + "finish_reason": "stop", + } + ], + } + + yield f"data: {json.dumps(finish_message)}\n\n" + yield f"data: [DONE]" + + return StreamingResponse(stream_content(), media_type="text/event-stream") + else: + res = pipe( + user_message=user_message, + model_id=pipeline_id, + messages=messages, + body=form_data.model_dump(), + ) + logging.info(f"stream:false:{res}") + + if isinstance(res, dict): + return res + elif isinstance(res, BaseModel): + return res.model_dump() + else: + + message = "" + + if isinstance(res, str): + message = res + + if isinstance(res, Generator): + for stream in res: + message = f"{message}{stream}" + + logging.info(f"stream:false:{message}") + return { + "id": f"{form_data.model}-{str(uuid.uuid4())}", + "object": "chat.completion", + "created": int(time.time()), + "model": form_data.model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": message, + }, + "logprobs": None, + "finish_reason": "stop", + } + ], + } + + return await run_in_threadpool(job) diff --git a/openwebui/pipelines/pipelines/.gitignore b/openwebui/pipelines/pipelines/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/openwebui/pipelines/requirements-minimum.txt b/openwebui/pipelines/requirements-minimum.txt new file mode 100644 index 0000000..559be6a --- /dev/null +++ b/openwebui/pipelines/requirements-minimum.txt @@ -0,0 +1,15 @@ +fastapi==0.111.0 +uvicorn[standard]==0.22.0 +pydantic==2.7.1 +python-multipart==0.0.9 +python-socketio +grpcio + +passlib==1.7.4 +passlib[bcrypt] +PyJWT[crypto] + +requests==2.32.2 +aiohttp==3.9.5 +httpx + diff --git a/openwebui/pipelines/requirements.txt b/openwebui/pipelines/requirements.txt new file mode 100644 index 0000000..b3cc96e --- /dev/null +++ b/openwebui/pipelines/requirements.txt @@ -0,0 +1,67 @@ +fastapi==0.111.0 +uvicorn[standard]==0.22.0 +pydantic==2.7.1 +python-multipart==0.0.9 +python-socketio +grpcio + +passlib==1.7.4 +passlib[bcrypt] +PyJWT[crypto] + +requests==2.32.2 +aiohttp==3.9.5 +httpx + +# AI libraries +openai +anthropic +google-generativeai +vertexai + +# Database +pymongo +peewee +SQLAlchemy +boto3 +redis +sqlmodel +chromadb +psycopg2-binary + +# Observability +langfuse +ddtrace +opik + +# ML libraries +torch +numpy +pandas + +xgboost +scikit-learn + +# NLP libraries +sentence-transformers +transformers +tokenizers +nltk +tiktoken + +# Image processing +Pillow +opencv-python + +# Visualization +matplotlib +seaborn + +# Web scraping +selenium +playwright +beautifulsoup4 + +# Llama Index for RAG +llama-index +llama-index-llms-ollama \ No newline at end of file diff --git a/openwebui/pipelines/schemas.py b/openwebui/pipelines/schemas.py new file mode 100644 index 0000000..caa0342 --- /dev/null +++ b/openwebui/pipelines/schemas.py @@ -0,0 +1,22 @@ +from typing import List, Optional +from pydantic import BaseModel, ConfigDict + +class OpenAIChatMessage(BaseModel): + role: str + content: str | List + + model_config = ConfigDict(extra="allow") + + +class OpenAIChatCompletionForm(BaseModel): + stream: bool = True + model: str + messages: List[OpenAIChatMessage] + + model_config = ConfigDict(extra="allow") + + +class FilterForm(BaseModel): + body: dict + user: Optional[dict] = None + model_config = ConfigDict(extra="allow") \ No newline at end of file diff --git a/openwebui/pipelines/start.bat b/openwebui/pipelines/start.bat new file mode 100644 index 0000000..248325e --- /dev/null +++ b/openwebui/pipelines/start.bat @@ -0,0 +1,5 @@ +@echo off +set PORT=9099 +set HOST=0.0.0.0 + +uvicorn main:app --host %HOST% --port %PORT% --forwarded-allow-ips '*' \ No newline at end of file diff --git a/openwebui/pipelines/start.sh b/openwebui/pipelines/start.sh new file mode 100644 index 0000000..e175741 --- /dev/null +++ b/openwebui/pipelines/start.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +PORT="${PORT:-9099}" +HOST="${HOST:-0.0.0.0}" +# Default value for PIPELINES_DIR +PIPELINES_DIR=${PIPELINES_DIR:-./pipelines} + +UVICORN_LOOP="${UVICORN_LOOP:-auto}" + +# Function to reset pipelines +reset_pipelines_dir() { + if [ "$RESET_PIPELINES_DIR" = true ]; then + echo "Resetting pipelines directory: $PIPELINES_DIR" + + # Check if the directory exists + if [ -d "$PIPELINES_DIR" ]; then + # Remove all contents of the directory + rm -rf "${PIPELINES_DIR:?}"/* + echo "All contents in $PIPELINES_DIR have been removed." + + # Optionally recreate the directory if needed + mkdir -p "$PIPELINES_DIR" + echo "$PIPELINES_DIR has been recreated." + else + echo "Directory $PIPELINES_DIR does not exist. No action taken." + fi + else + echo "RESET_PIPELINES_DIR is not set to true. No action taken." + fi +} + +# Function to install requirements if requirements.txt is provided +install_requirements() { + if [[ -f "$1" ]]; then + echo "requirements.txt found at $1. Installing requirements..." + pip install -r "$1" + else + echo "requirements.txt not found at $1. Skipping installation of requirements." + fi +} + +# Check if the PIPELINES_REQUIREMENTS_PATH environment variable is set and non-empty +if [[ -n "$PIPELINES_REQUIREMENTS_PATH" ]]; then + # Install requirements from the specified requirements.txt + install_requirements "$PIPELINES_REQUIREMENTS_PATH" +else + echo "PIPELINES_REQUIREMENTS_PATH not specified. Skipping installation of requirements." +fi + + +# Function to download the pipeline files +download_pipelines() { + local path=$1 + local destination=$2 + + # Remove any surrounding quotes from the path + path=$(echo "$path" | sed 's/^"//;s/"$//') + + echo "Downloading pipeline files from $path to $destination..." + + if [[ "$path" =~ ^https://github.com/.*/.*/blob/.* ]]; then + # It's a single file + dest_file=$(basename "$path") + curl -L "$path?raw=true" -o "$destination/$dest_file" + elif [[ "$path" =~ ^https://github.com/.*/.*/tree/.* ]]; then + # It's a folder + git_repo=$(echo "$path" | awk -F '/tree/' '{print $1}') + subdir=$(echo "$path" | awk -F '/tree/' '{print $2}') + git clone --depth 1 --filter=blob:none --sparse "$git_repo" "$destination" + ( + cd "$destination" || exit + git sparse-checkout set "$subdir" + ) + elif [[ "$path" =~ \.py$ ]]; then + # It's a single .py file (but not from GitHub) + dest_file=$(basename "$path") + curl -L "$path" -o "$destination/$dest_file" + else + echo "Invalid URL format: $path" + exit 1 + fi +} + +# Function to parse and install requirements from frontmatter +install_frontmatter_requirements() { + local file=$1 + local file_content=$(cat "$1") + # Extract the first triple-quoted block + local first_block=$(echo "$file_content" | awk '/"""/{flag=!flag; if(flag) count++; if(count == 2) {exit}} flag' ) + + # Check if the block contains requirements + local requirements=$(echo "$first_block" | grep -i 'requirements:') + + if [ -n "$requirements" ]; then + # Extract the requirements list + requirements=$(echo "$requirements" | awk -F': ' '{print $2}' | tr ',' ' ' | tr -d '\r') + + # Construct and echo the pip install command + local pip_command="pip install $requirements" + echo "$pip_command" + pip install $requirements + else + echo "No requirements found in frontmatter of $file." + fi +} + + +# Parse command line arguments for mode +MODE="full" # select a runmode ("setup", "run", "full" (setup + run)) +while [[ "$#" -gt 0 ]]; do + case $1 in + --mode) MODE="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done +if [[ "$MODE" != "setup" && "$MODE" != "run" && "$MODE" != "full" ]]; then + echo "Invalid script mode: $MODE" + echo " Example usage: './start.sh --mode [setup|run|full]' " + exit 1 +fi + +# Function to handle different modes, added 1/29/24 +if [[ "$MODE" == "setup" || "$MODE" == "full" ]]; then + echo "Download + install Executed in mode: $MODE" + + reset_pipelines_dir + if [[ -n "$PIPELINES_REQUIREMENTS_PATH" ]]; then + install_requirements "$PIPELINES_REQUIREMENTS_PATH" + else + echo "PIPELINES_REQUIREMENTS_PATH not specified. Skipping installation of requirements." + fi + + if [[ -n "$PIPELINES_URLS" ]]; then + if [ ! -d "$PIPELINES_DIR" ]; then + mkdir -p "$PIPELINES_DIR" + fi + + IFS=';' read -ra ADDR <<< "$PIPELINES_URLS" + for path in "${ADDR[@]}"; do + download_pipelines "$path" "$PIPELINES_DIR" + done + + for file in "$PIPELINES_DIR"/*; do + if [[ -f "$file" ]]; then + install_frontmatter_requirements "$file" + fi + done + else + echo "PIPELINES_URLS not specified. Skipping pipelines download and installation." + fi +fi + +if [[ "$MODE" == "run" || "$MODE" == "full" ]]; then + echo "Running via Mode: $MODE" + uvicorn main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' --loop "$UVICORN_LOOP" +fi + diff --git a/openwebui/pipelines/utils/pipelines/auth.py b/openwebui/pipelines/utils/pipelines/auth.py new file mode 100644 index 0000000..2b09820 --- /dev/null +++ b/openwebui/pipelines/utils/pipelines/auth.py @@ -0,0 +1,77 @@ +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi import HTTPException, status, Depends + + +from pydantic import BaseModel +from typing import Union, Optional + + +from passlib.context import CryptContext +from datetime import datetime, timedelta +import jwt +import logging +import os + +import requests +import uuid + + +from config import API_KEY, PIPELINES_DIR + + +SESSION_SECRET = os.getenv("SESSION_SECRET", " ") +ALGORITHM = "HS256" + +############## +# Auth Utils +############## + +bearer_security = HTTPBearer() +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + + +def verify_password(plain_password, hashed_password): + return ( + pwd_context.verify(plain_password, hashed_password) if hashed_password else None + ) + + +def get_password_hash(password): + return pwd_context.hash(password) + + +def create_token(data: dict, expires_delta: Union[timedelta, None] = None) -> str: + payload = data.copy() + + if expires_delta: + expire = datetime.utcnow() + expires_delta + payload.update({"exp": expire}) + + encoded_jwt = jwt.encode(payload, SESSION_SECRET, algorithm=ALGORITHM) + return encoded_jwt + + +def decode_token(token: str) -> Optional[dict]: + try: + decoded = jwt.decode(token, SESSION_SECRET, algorithms=[ALGORITHM]) + return decoded + except Exception as e: + return None + + +def extract_token_from_auth_header(auth_header: str): + return auth_header[len("Bearer ") :] + + +def get_current_user( + credentials: HTTPAuthorizationCredentials = Depends(bearer_security), +) -> Optional[dict]: + token = credentials.credentials + + if token != API_KEY: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid API key", + ) + + return token diff --git a/openwebui/pipelines/utils/pipelines/main.py b/openwebui/pipelines/utils/pipelines/main.py new file mode 100644 index 0000000..5d33522 --- /dev/null +++ b/openwebui/pipelines/utils/pipelines/main.py @@ -0,0 +1,153 @@ +import uuid +import time + +from typing import List +from schemas import OpenAIChatMessage + +import inspect +from typing import get_type_hints, Literal, Tuple + + +def stream_message_template(model: str, message: str): + return { + "id": f"{model}-{str(uuid.uuid4())}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": {"content": message}, + "logprobs": None, + "finish_reason": None, + } + ], + } + + +def get_last_user_message(messages: List[dict]) -> str: + for message in reversed(messages): + if message["role"] == "user": + if isinstance(message["content"], list): + for item in message["content"]: + if item["type"] == "text": + return item["text"] + return message["content"] + return None + + +def get_last_assistant_message(messages: List[dict]) -> str: + for message in reversed(messages): + if message["role"] == "assistant": + if isinstance(message["content"], list): + for item in message["content"]: + if item["type"] == "text": + return item["text"] + return message["content"] + return None + + +def get_system_message(messages: List[dict]) -> dict: + for message in messages: + if message["role"] == "system": + return message + return None + + +def remove_system_message(messages: List[dict]) -> List[dict]: + return [message for message in messages if message["role"] != "system"] + + +def pop_system_message(messages: List[dict]) -> Tuple[dict, List[dict]]: + return get_system_message(messages), remove_system_message(messages) + + +def add_or_update_system_message(content: str, messages: List[dict]) -> List[dict]: + """ + Adds a new system message at the beginning of the messages list + or updates the existing system message at the beginning. + + :param msg: The message to be added or appended. + :param messages: The list of message dictionaries. + :return: The updated list of message dictionaries. + """ + + if messages and messages[0].get("role") == "system": + messages[0]["content"] += f"{content}\n{messages[0]['content']}" + else: + # Insert at the beginning + messages.insert(0, {"role": "system", "content": content}) + + return messages + + +def doc_to_dict(docstring): + lines = docstring.split("\n") + description = lines[1].strip() + param_dict = {} + + for line in lines: + if ":param" in line: + line = line.replace(":param", "").strip() + param, desc = line.split(":", 1) + param_dict[param.strip()] = desc.strip() + ret_dict = {"description": description, "params": param_dict} + return ret_dict + + +def get_tools_specs(tools) -> List[dict]: + function_list = [ + {"name": func, "function": getattr(tools, func)} + for func in dir(tools) + if callable(getattr(tools, func)) and not func.startswith("__") + ] + + specs = [] + + for function_item in function_list: + function_name = function_item["name"] + function = function_item["function"] + + function_doc = doc_to_dict(function.__doc__ or function_name) + specs.append( + { + "name": function_name, + # TODO: multi-line desc? + "description": function_doc.get("description", function_name), + "parameters": { + "type": "object", + "properties": { + param_name: { + "type": param_annotation.__name__.lower(), + **( + { + "enum": ( + param_annotation.__args__ + if hasattr(param_annotation, "__args__") + else None + ) + } + if hasattr(param_annotation, "__args__") + else {} + ), + "description": function_doc.get("params", {}).get( + param_name, param_name + ), + } + for param_name, param_annotation in get_type_hints( + function + ).items() + if param_name != "return" + }, + "required": [ + name + for name, param in inspect.signature( + function + ).parameters.items() + if param.default is param.empty + ], + }, + } + ) + + return specs diff --git a/openwebui/pipelines/utils/pipelines/misc.py b/openwebui/pipelines/utils/pipelines/misc.py new file mode 100644 index 0000000..e0c1e1c --- /dev/null +++ b/openwebui/pipelines/utils/pipelines/misc.py @@ -0,0 +1,35 @@ +import re + + +def convert_to_raw_url(github_url): + """ + Converts a GitHub URL to a raw URL. + + Example: + https://github.com/user/repo/blob/branch/path/to/file.ext + becomes + https://raw.githubusercontent.com/user/repo/branch/path/to/file.ext + + Parameters: + github_url (str): The GitHub URL to convert. + + Returns: + str: The converted raw URL. + """ + # Define the regular expression pattern + pattern = r"https://github\.com/(.+?)/(.+?)/blob/(.+?)/(.+)" + + # Use the pattern to match and extract parts of the URL + match = re.match(pattern, github_url) + + if match: + user_repo = match.group(1) + "/" + match.group(2) + branch = match.group(3) + file_path = match.group(4) + + # Construct the raw URL + raw_url = f"https://raw.githubusercontent.com/{user_repo}/{branch}/{file_path}" + return raw_url + + # If the URL does not match the expected pattern, return the original URL or raise an error + return github_url diff --git a/src/config/settings.py b/src/config/settings.py index 10f43e9..80f05ab 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -61,6 +61,7 @@ def llm_config(self) -> Dict[str, Any]: "base_url": self.LITELLM_BASE_URL, "api_key": self.LITELLM_API_KEY, "model": self.LITELLM_MODEL, + "model_kwargs": {"stream_options": {"include_usage": True}}, }