diff --git a/.github/workflows/check-all-models.yml b/.github/workflows/check-all-models.yml new file mode 100644 index 0000000..9c9279c --- /dev/null +++ b/.github/workflows/check-all-models.yml @@ -0,0 +1,123 @@ +name: Check All Models + +on: + workflow_dispatch: + inputs: + models: + description: 'Specific models to test (space-separated, leave empty for all)' + required: false + type: string + schedule: + # Run daily at 6:00 UTC + - cron: '0 6 * * *' + push: + branches: + - main + paths: + - 'check_models.py' + - 'check_models.js' + - 'requirements.txt' + - 'package.json' + - '.github/workflows/check-all-models.yml' + +jobs: + check-models-python: + name: Python Model Check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Check all models + if: github.event.inputs.models == '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: python check_models.py + + - name: Check specific models + if: github.event.inputs.models != '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: python check_models.py ${{ github.event.inputs.models }} + + - name: Upload Python results + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-model-check-results + path: | + *.log + retention-days: 30 + + check-models-javascript: + name: JavaScript Model Check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Check all models + if: github.event.inputs.models == '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: node check_models.js + + - name: Check specific models + if: github.event.inputs.models != '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: node check_models.js ${{ github.event.inputs.models }} + + - name: Upload JavaScript results + if: always() + uses: actions/upload-artifact@v4 + with: + name: javascript-model-check-results + path: | + *.log + retention-days: 30 + + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [check-models-python, check-models-javascript] + if: always() + + steps: + - name: Check job results + run: | + echo "Python Check: ${{ needs.check-models-python.result }}" + echo "JavaScript Check: ${{ needs.check-models-javascript.result }}" + + if [ "${{ needs.check-models-python.result }}" != "success" ] || [ "${{ needs.check-models-javascript.result }}" != "success" ]; then + echo "⚠️ One or more model checks failed" + exit 1 + else + echo "✅ All model checks passed" + fi diff --git a/.github/workflows/check-models-javascript.yml b/.github/workflows/check-models-javascript.yml new file mode 100644 index 0000000..556be2c --- /dev/null +++ b/.github/workflows/check-models-javascript.yml @@ -0,0 +1,59 @@ +name: Check Models - JavaScript + +on: + workflow_dispatch: + inputs: + models: + description: 'Specific models to test (space-separated, leave empty for all)' + required: false + type: string + schedule: + # Run daily at 6:00 UTC + - cron: '0 6 * * *' + push: + branches: + - main + paths: + - 'check_models.js' + - 'package.json' + - '.github/workflows/check-models-javascript.yml' + +jobs: + check-models: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Check all models + if: github.event.inputs.models == '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: node check_models.js + + - name: Check specific models + if: github.event.inputs.models != '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: node check_models.js ${{ github.event.inputs.models }} + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: javascript-model-check-results + path: | + *.log + retention-days: 30 diff --git a/.github/workflows/check-models-python.yml b/.github/workflows/check-models-python.yml new file mode 100644 index 0000000..1b3bbd7 --- /dev/null +++ b/.github/workflows/check-models-python.yml @@ -0,0 +1,61 @@ +name: Check Models - Python + +on: + workflow_dispatch: + inputs: + models: + description: 'Specific models to test (space-separated, leave empty for all)' + required: false + type: string + schedule: + # Run daily at 6:00 UTC + - cron: '0 6 * * *' + push: + branches: + - main + paths: + - 'check_models.py' + - 'requirements.txt' + - '.github/workflows/check-models-python.yml' + +jobs: + check-models: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Check all models + if: github.event.inputs.models == '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: python check_models.py + + - name: Check specific models + if: github.event.inputs.models != '' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1' }} + run: python check_models.py ${{ github.event.inputs.models }} + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-model-check-results + path: | + *.log + retention-days: 30 diff --git a/README.md b/README.md index b993b11..1097a32 100644 --- a/README.md +++ b/README.md @@ -27,4 +27,128 @@ Each example directory contains: ## API Key Get your API key from the Telegram bot: https://t.me/DeepGPTBot -Use the `/api` command to obtain your key. \ No newline at end of file +Use the `/api` command to obtain your key. + +## Model Availability Testing + +This repository includes automated model availability checkers for both Python and JavaScript that can be run locally or via GitHub Actions. + +### Supported Models + +The following models are tested: +- `o3-mini`, `o1-preview`, `o1-mini` +- `gpt-4o`, `gpt-4o-mini`, `gpt-3.5-turbo`, `gpt-auto` +- `claude-3-opus`, `claude-3-5-sonnet`, `claude-3-5-haiku`, `claude-3-7-sonnet` +- `deepseek-chat`, `deepseek-reasoner` + +### Local Testing + +#### Python + +```bash +# Install dependencies +pip install -r requirements.txt + +# Set your API key +export OPENAI_API_KEY="your-api-key-here" + +# Test all models +python check_models.py + +# Test specific models +python check_models.py gpt-4o claude-3-5-sonnet +``` + +#### JavaScript + +```bash +# Install dependencies +npm install + +# Set your API key +export OPENAI_API_KEY="your-api-key-here" + +# Test all models +node check_models.js + +# Test specific models +node check_models.js gpt-4o claude-3-5-sonnet +``` + +### GitHub Actions + +The repository includes three GitHub Actions workflows for automated model testing: + +1. **Check Models - Python** (`.github/workflows/check-models-python.yml`) + - Tests models using Python + - Runs daily at 6:00 UTC + - Can be manually triggered with specific models + +2. **Check Models - JavaScript** (`.github/workflows/check-models-javascript.yml`) + - Tests models using JavaScript + - Runs daily at 6:00 UTC + - Can be manually triggered with specific models + +3. **Check All Models** (`.github/workflows/check-all-models.yml`) + - Tests models using both Python and JavaScript + - Runs daily at 6:00 UTC + - Provides a combined summary of results + +#### Manual Workflow Execution + +You can manually trigger any workflow from the GitHub Actions tab: + +1. Go to the **Actions** tab in your repository +2. Select the workflow you want to run +3. Click **Run workflow** +4. Optionally specify models to test (e.g., `gpt-4o claude-3-5-sonnet`) +5. Click **Run workflow** to start + +#### Required Secrets + +To use GitHub Actions, you need to configure the following secret: + +- `OPENAI_API_KEY` - Your Deep.Assistant API key (get from [@DeepGPTBot](https://t.me/DeepGPTBot) using `/api` command) + +#### Optional Variables + +- `OPENAI_API_BASE` - Custom API base URL (defaults to `https://api.deep.assistant.run.place/v1`) + +#### Setting up Secrets + +1. Go to your repository **Settings** +2. Navigate to **Secrets and variables** → **Actions** +3. Click **New repository secret** +4. Add `OPENAI_API_KEY` with your API key + +### Output Format + +The model checkers provide detailed output including: +- Total models tested +- Available models count +- Unavailable models count +- Detailed status for each model +- Error messages for failures + +Example output: +``` +============================================================ +MODEL AVAILABILITY TEST RESULTS +============================================================ + +Total models tested: 13 +Available models: 10 +Unavailable models: 3 + +✓ AVAILABLE MODELS: + - gpt-4o + - gpt-4o-mini + - claude-3-5-sonnet + ... + +✗ UNAVAILABLE MODELS: + - model-name: HTTP 404: Model not found + Error: The model 'model-name' does not exist + +============================================================ +``` \ No newline at end of file diff --git a/check_models.js b/check_models.js new file mode 100644 index 0000000..09dfe2d --- /dev/null +++ b/check_models.js @@ -0,0 +1,197 @@ +#!/usr/bin/env node +/** + * Model availability checker for GitHub Actions. + * Tests Deep.Assistant API models and reports their availability. + */ +import OpenAI from 'openai'; +import * as fs from 'fs'; + +// Model configurations +const MODELS = [ + "o3-mini", + "o1-preview", + "o1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-3.5-turbo", + "gpt-auto", + "claude-3-opus", + "claude-3-5-sonnet", + "claude-3-5-haiku", + "claude-3-7-sonnet", + "deepseek-chat", + "deepseek-reasoner", +]; + +/** + * Test a single model for availability + */ +async function testModel(client, model) { + const result = { + model, + available: false, + status: '', + error: null + }; + + try { + const completion = await client.chat.completions.create({ + messages: [{ role: 'user', content: 'Test' }], + model, + max_tokens: 10, + }); + + const responseModel = completion.model; + + // Check if the returned model matches the requested model + if (responseModel === model) { + result.available = true; + result.status = 'Available'; + } else { + result.status = `Wrong model returned: ${responseModel}`; + result.error = `Expected ${model}, got ${responseModel}`; + } + } catch (error) { + result.status = 'Error'; + + if (error.status) { + result.status = `HTTP ${error.status}`; + result.error = error.message || `HTTP error ${error.status}`; + } else if (error.code === 'ETIMEDOUT' || error.message?.includes('timeout')) { + result.status = 'Timeout'; + result.error = 'Request timed out'; + } else { + result.error = error.message || String(error); + } + } + + return result; +} + +/** + * Test all models or a specific subset + */ +async function testAllModels(apiKey, modelsToTest = null) { + const models = modelsToTest || MODELS; + + console.log(`Testing ${models.length} model(s)...`); + + const client = new OpenAI({ + apiKey, + baseURL: process.env.OPENAI_API_BASE || 'https://api.deep.assistant.run.place/v1', + timeout: 60000, // 60 second timeout + }); + + // Test all models in parallel + const results = await Promise.all( + models.map(model => testModel(client, model)) + ); + + // Organize results + const available = []; + const unavailable = []; + + for (const result of results) { + if (result.available) { + available.push(result.model); + } else { + unavailable.push({ + model: result.model, + status: result.status, + error: result.error + }); + } + } + + return { + total: models.length, + available, + unavailable, + results + }; +} + +/** + * Print test results in a readable format + */ +function printResults(summary) { + console.log('\n' + '='.repeat(60)); + console.log('MODEL AVAILABILITY TEST RESULTS'); + console.log('='.repeat(60)); + + console.log(`\nTotal models tested: ${summary.total}`); + console.log(`Available models: ${summary.available.length}`); + console.log(`Unavailable models: ${summary.unavailable.length}`); + + if (summary.available.length > 0) { + console.log('\n✓ AVAILABLE MODELS:'); + for (const model of summary.available) { + console.log(` - ${model}`); + } + } + + if (summary.unavailable.length > 0) { + console.log('\n✗ UNAVAILABLE MODELS:'); + for (const item of summary.unavailable) { + console.log(` - ${item.model}: ${item.status}`); + if (item.error) { + console.log(` Error: ${item.error}`); + } + } + } + + console.log('\n' + '='.repeat(60)); +} + +/** + * Main entry point + */ +async function main() { + // Get API key from environment + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + console.error('Error: OPENAI_API_KEY environment variable not set'); + process.exit(1); + } + + // Parse command line arguments for specific models + const modelsToTest = process.argv.length > 2 ? process.argv.slice(2) : MODELS; + + if (process.argv.length > 2) { + console.log(`Testing specific models: ${modelsToTest.join(', ')}`); + } + + // Run tests + const summary = await testAllModels(apiKey, modelsToTest); + + // Print results + printResults(summary); + + // Output for GitHub Actions + if (process.env.GITHUB_ACTIONS) { + const outputFile = process.env.GITHUB_OUTPUT; + if (outputFile) { + const output = [ + `available_count=${summary.available.length}`, + `unavailable_count=${summary.unavailable.length}`, + `total_count=${summary.total}` + ].join('\n') + '\n'; + + fs.appendFileSync(outputFile, output); + } + } + + // Exit with error code if any models are unavailable + if (summary.unavailable.length > 0) { + console.log(`\n⚠ Warning: ${summary.unavailable.length} model(s) unavailable`); + process.exit(1); + } else { + console.log('\n✓ All models are available!'); + process.exit(0); + } +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/check_models.py b/check_models.py new file mode 100644 index 0000000..a06ab3e --- /dev/null +++ b/check_models.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +Model availability checker for GitHub Actions. +Tests Deep.Assistant API models and reports their availability. +""" +import os +import sys +import json +import asyncio +import aiohttp +from typing import Dict, List, Any + + +# Model configurations +MODELS = [ + "o3-mini", + "o1-preview", + "o1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-3.5-turbo", + "gpt-auto", + "claude-3-opus", + "claude-3-5-sonnet", + "claude-3-5-haiku", + "claude-3-7-sonnet", + "deepseek-chat", + "deepseek-reasoner", +] + + +async def test_model(session: aiohttp.ClientSession, model: str, api_key: str) -> Dict[str, Any]: + """Test a single model for availability.""" + url = f"{os.getenv('OPENAI_API_BASE', 'https://api.deep.assistant.run.place/v1')}/chat/completions" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "messages": [{"role": "user", "content": "Test"}], + "model": model, + "max_tokens": 10 + } + + result = { + "model": model, + "available": False, + "status": "", + "error": None + } + + try: + async with session.post( + url, + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + data = await response.json() + response_model = data.get("model", "") + + # Check if the returned model matches the requested model + if response_model == model: + result["available"] = True + result["status"] = "Available" + else: + result["status"] = f"Wrong model returned: {response_model}" + result["error"] = f"Expected {model}, got {response_model}" + else: + result["status"] = f"HTTP {response.status}" + try: + error_data = await response.json() + if 'error' in error_data and 'message' in error_data['error']: + result["error"] = error_data['error']['message'] + result["status"] = f"HTTP {response.status}: {result['error']}" + except: + result["error"] = f"HTTP error {response.status}" + + except asyncio.TimeoutError: + result["status"] = "Timeout" + result["error"] = "Request timed out after 60 seconds" + except Exception as e: + result["status"] = "Error" + result["error"] = str(e) + + return result + + +async def test_all_models(api_key: str, models: List[str] = None) -> Dict[str, Any]: + """Test all models or a specific subset.""" + if models is None: + models = MODELS + + print(f"Testing {len(models)} model(s)...") + + async with aiohttp.ClientSession() as session: + tasks = [test_model(session, model, api_key) for model in models] + results = await asyncio.gather(*tasks) + + # Organize results + available = [] + unavailable = [] + + for result in results: + if result["available"]: + available.append(result["model"]) + else: + unavailable.append({ + "model": result["model"], + "status": result["status"], + "error": result["error"] + }) + + return { + "total": len(models), + "available": available, + "unavailable": unavailable, + "results": results + } + + +def print_results(summary: Dict[str, Any]) -> None: + """Print test results in a readable format.""" + print("\n" + "=" * 60) + print("MODEL AVAILABILITY TEST RESULTS") + print("=" * 60) + + print(f"\nTotal models tested: {summary['total']}") + print(f"Available models: {len(summary['available'])}") + print(f"Unavailable models: {len(summary['unavailable'])}") + + if summary['available']: + print("\n✓ AVAILABLE MODELS:") + for model in summary['available']: + print(f" - {model}") + + if summary['unavailable']: + print("\n✗ UNAVAILABLE MODELS:") + for item in summary['unavailable']: + print(f" - {item['model']}: {item['status']}") + if item['error']: + print(f" Error: {item['error']}") + + print("\n" + "=" * 60) + + +def main(): + """Main entry point.""" + # Get API key from environment + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + print("Error: OPENAI_API_KEY environment variable not set", file=sys.stderr) + sys.exit(1) + + # Parse command line arguments for specific models + models_to_test = MODELS + if len(sys.argv) > 1: + # Test specific models passed as arguments + models_to_test = sys.argv[1:] + print(f"Testing specific models: {', '.join(models_to_test)}") + + # Run tests + summary = asyncio.run(test_all_models(api_key, models_to_test)) + + # Print results + print_results(summary) + + # Output JSON for GitHub Actions + if os.getenv("GITHUB_ACTIONS"): + output_file = os.getenv("GITHUB_OUTPUT") + if output_file: + with open(output_file, "a") as f: + f.write(f"available_count={len(summary['available'])}\n") + f.write(f"unavailable_count={len(summary['unavailable'])}\n") + f.write(f"total_count={summary['total']}\n") + + # Exit with error code if any models are unavailable + if summary['unavailable']: + print(f"\n⚠ Warning: {len(summary['unavailable'])} model(s) unavailable") + sys.exit(1) + else: + print("\n✓ All models are available!") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/package.json b/package.json new file mode 100644 index 0000000..4b48cc6 --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "name": "api-tester-model-checker", + "version": "1.0.0", + "description": "Model availability checker for Deep.Assistant API", + "type": "module", + "main": "check_models.js", + "scripts": { + "check": "node check_models.js" + }, + "dependencies": { + "openai": "^4.0.0" + }, + "keywords": ["openai", "api", "testing", "models"], + "author": "Deep.Assistant", + "license": "Unlicense" +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3beb7cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +aiohttp>=3.9.0