diff --git a/CHANGELOG.md b/CHANGELOG.md index 77430ca2e..79321a040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Improve regex of denied commands in plan agent. +- Fix fetch models regression. #299 ## 0.100.2 diff --git a/src/eca/llm_providers/anthropic.clj b/src/eca/llm_providers/anthropic.clj index 99e6065c9..584594f8c 100644 --- a/src/eca/llm_providers/anthropic.clj +++ b/src/eca/llm_providers/anthropic.clj @@ -208,7 +208,7 @@ tools web-search extra-payload extra-headers supports-image? http-client]} {:keys [on-message-received on-error on-reason on-prepare-tool-call on-tools-called on-usage-updated] :as callbacks}] (let [messages (-> (concat (normalize-messages past-messages supports-image?) - (normalize-messages (fix-non-thinking-assistant-messages user-messages) supports-image?)) + (normalize-messages (fix-non-thinking-assistant-messages user-messages) supports-image?)) merge-adjacent-assistants) stream? (boolean callbacks) body (deep-merge diff --git a/src/eca/models.clj b/src/eca/models.clj index 726e647dd..c3d86f538 100644 --- a/src/eca/models.clj +++ b/src/eca/models.clj @@ -15,6 +15,27 @@ (def ^:private models-dev-api-url "https://models.dev/api.json") (def ^:private models-dev-timeout-ms 5000) +(def ^:private provider-models-timeout-ms 10000) + +;; Provider API types that support native /models endpoint fetching +(def ^:private native-models-endpoint-providers + #{"anthropic" "openai" "openai-chat" "openai-responses"}) + +(defn ^:private provider-models-endpoint-path + "Returns the appropriate /models endpoint path for a given provider API type." + [api-type] + (case api-type + "anthropic" "/v1/models" + ("openai" "openai-responses") "/v1/models" + "openai-chat" "/models" + nil)) + +(defn ^:private models-endpoint-headers + [api-key] + (client/merge-llm-headers + (assoc-some + {"Content-Type" "application/json"} + "Authorization" (when api-key (str "Bearer " api-key))))) (defn ^:private fetch-models-dev-data [] (let [{:keys [status body]} (http/get models-dev-api-url @@ -27,14 +48,9 @@ (throw (ex-info (format "models.dev request failed with status %s" status) {:status status}))))) -;; clojure.core/memoize does NOT cache thrown exceptions. -;; If fetch throws, the next call will retry the HTTP request. -;; Once it succeeds, the result is cached for the process lifetime. -(def ^:private models-dev-fetch-memoized (memoize fetch-models-dev-data)) - (defn ^:private models-dev [] (try - (let [data (models-dev-fetch-memoized)] + (let [data (fetch-models-dev-data)] (if (map? data) data (do @@ -68,7 +84,7 @@ (defn ^:private all "Return all known existing models with their capabilities and configs." - [] + [models-dev-data] (reduce (fn [m [provider provider-config]] (merge m @@ -92,7 +108,7 @@ {} (get provider-config "models")))) {} - (models-dev))) + models-dev-data)) (defn ^:private auth-valid? [full-model db config] (let [[provider _model] (string/split full-model #"/" 2)] @@ -138,15 +154,19 @@ (when-let [provider-by-id (get by-id provider)] (models-dev-provider-without-api? provider-by-id)))) +(defn ^:private fetch-model-catalog-enabled? + [provider-config] + (boolean + (and (:api provider-config) + (not= false (:fetchModels provider-config))))) + (defn ^:private add-models-from-models-dev? "Returns true when provider should load model catalog from models.dev. Opt-out with fetchModels=false." [provider provider-config config models-dev-index] - (let [provider-api-url (llm-util/provider-api-url provider config) - fetch-models (:fetchModels provider-config)] + (let [provider-api-url (llm-util/provider-api-url provider config)] (boolean - (and (:api provider-config) - (not= false fetch-models) + (and (fetch-model-catalog-enabled? provider-config) (resolve-models-dev-provider provider provider-api-url models-dev-index))))) (defn ^:private deprecated-model? @@ -166,6 +186,61 @@ (format "Provider '%s': Ignoring models.dev model entry '%s' with invalid key/model fields" provider model-key))) +(defn ^:private fetch-provider-native-models + "Fetches models from provider's native /models endpoint. + Returns a map of model-id -> {} on success, nil on failure." + [{:keys [api-url api-key api-type provider]}] + (when-let [models-path (provider-models-endpoint-path api-type)] + (let [url (shared/join-api-url api-url models-path) + rid (llm-util/gen-rid) + headers (models-endpoint-headers api-key)] + (try + (llm-util/log-request logger-tag rid url nil headers) + (let [{:keys [status body]} (http/get url + {:headers headers + :throw-exceptions? false + :as :json + :http-client (client/merge-with-global-http-client {}) + :timeout provider-models-timeout-ms})] + (if (not= 200 status) + (logger/warn logger-tag + (format "Provider '%s': /models endpoint returned status %s" + provider status)) + + (do + (llm-util/log-response logger-tag rid "models" body) + (let [models-data (:data body)] + (if (not (sequential? models-data)) + (logger/warn logger-tag + (format "Provider '%s': /models payload missing sequential :data (status %s, keys %s)" + provider status (if (map? body) (-> body keys sort vec) :non-map-body))) + (zipmap (keep :id models-data) (repeat {}))))))) + (catch Exception e + (logger/warn logger-tag + (format "Provider '%s': Failed to fetch models from %s: %s" + provider url (ex-message e)))))))) + +(defn ^:private fetch-provider-native-models-with-fallback + "Tries to fetch models from provider's native endpoint first. + Returns a map of model-id -> model-config map, or nil." + [provider provider-config config db] + (when (contains? native-models-endpoint-providers (:api provider-config)) + (let [api-url (llm-util/provider-api-url provider config) + [_ api-key] (llm-util/provider-api-key provider + (get-in db [:auth provider]) + config) + api-type (:api provider-config)] + (when (and api-url api-key) + (when-let [models (fetch-provider-native-models + {:provider provider + :api-url api-url + :api-key api-key + :api-type api-type})] + (logger/info logger-tag + (format "Provider '%s': Discovered %d models from native /models endpoint" + provider (count models))) + models))))) + (defn ^:private parse-models-dev-provider-models "Builds provider model config map from models.dev payload. Uses models.dev model key for selection." @@ -193,35 +268,25 @@ {} provider-models)))) -(defn ^:private fetch-models-dev-provider-models - "Loads models from models.dev for providers with matching API URL. - Fallbacks to provider id key when URL is unavailable in models.dev. - Returns a map of {provider-name -> {model-name -> model-config}}." - [config] - (let [models-dev-data (models-dev) - models-dev-index (models-dev-provider-index models-dev-data)] - (reduce - (fn [acc [provider provider-config]] - (if-not (add-models-from-models-dev? provider provider-config config models-dev-index) - acc - (let [provider-api-url (llm-util/provider-api-url provider config) - models-dev-provider (resolve-models-dev-provider - provider provider-api-url models-dev-index) - provider-models (some->> (get models-dev-provider "models") - (parse-models-dev-provider-models provider))] - (when (using-models-dev-provider-id-fallback? provider provider-api-url models-dev-index) - (logger/info logger-tag - (format "Provider '%s': Using models.dev provider-id fallback (url '%s' not matched)" - provider provider-api-url))) - (if provider-models - (do - (logger/info logger-tag - (format "Provider '%s': Loaded %d models from models.dev" - provider (count provider-models))) - (assoc acc provider provider-models)) - acc)))) - {} - (:providers config)))) +(defn ^:private fetch-single-provider-models-dev + "Fetches models from models.dev for a single provider. + Returns {model-name -> model-config} or nil if not found." + [provider provider-config config models-dev-index] + (when (add-models-from-models-dev? provider provider-config config models-dev-index) + (let [provider-api-url (llm-util/provider-api-url provider config) + models-dev-provider (resolve-models-dev-provider + provider provider-api-url models-dev-index) + provider-models (some->> (get models-dev-provider "models") + (parse-models-dev-provider-models provider))] + (when (using-models-dev-provider-id-fallback? provider provider-api-url models-dev-index) + (logger/info logger-tag + (format "Provider '%s': Using models.dev provider-id fallback (url '%s' not matched)" + provider provider-api-url))) + (when provider-models + (logger/info logger-tag + (format "Provider '%s': Loaded %d models from models.dev" + provider (count provider-models)))) + provider-models))) (defn ^:private build-model-capabilities "Build capabilities for a single model, looking up from known models database." @@ -256,16 +321,49 @@ [static-models dynamic-models] (merge-with merge dynamic-models static-models)) +(defn ^:private fetch-provider-models-with-priority + "Fetches models for all providers, trying native endpoint first, then models.dev. + Returns a map of {provider-name -> {model-name -> model-config}}." + ([config db] + (fetch-provider-models-with-priority config db (models-dev))) + ([config db models-dev-data] + (let [models-dev-index (models-dev-provider-index models-dev-data) + start-ns (System/nanoTime) + futures (into [] + (keep (fn [[provider provider-config]] + (when (fetch-model-catalog-enabled? provider-config) + [provider + (future + (or (fetch-provider-native-models-with-fallback + provider provider-config config db) + (fetch-single-provider-models-dev + provider provider-config config models-dev-index)))]))) + (:providers config)) + result (reduce + (fn [acc [provider f]] + (if-let [models @f] + (assoc acc provider models) + acc)) + {} + futures) + elapsed-ms (/ (- (System/nanoTime) start-ns) 1e6)] + (logger/info logger-tag + (format "Fetched model catalogs from %d providers in %.1fms" + (count result) elapsed-ms)) + result))) + (defn ^:private fetch-provider-model-catalogs - [config] - {:models-dev (fetch-models-dev-provider-models config)}) + ([config db] + (fetch-provider-model-catalogs config db (models-dev))) + ([config db models-dev-data] + {:models (fetch-provider-models-with-priority config db models-dev-data)})) (defn ^:private build-all-supported-models - [known-models config models-dev-provider-models] + [known-models config discovered-provider-models] (reduce (fn [p [provider provider-config]] (let [static-models (:models provider-config) - dynamic-models (get models-dev-provider-models provider) + dynamic-models (get discovered-provider-models provider) merged-models (merge-provider-models static-models dynamic-models)] (merge p (reduce @@ -279,13 +377,14 @@ (:providers config))) (defn sync-models! [db* config on-models-updated] - (let [known-models (all) + (let [models-dev-data (models-dev) + known-models (all models-dev-data) db @db* - {:keys [models-dev]} (fetch-provider-model-catalogs config) + {:keys [models]} (fetch-provider-model-catalogs config db models-dev-data) all-supported-models (build-all-supported-models known-models config - models-dev) + models) ollama-api-url (llm-util/provider-api-url "ollama" config) ollama-models (mapv (fn [{:keys [model] :as ollama-model}] @@ -310,7 +409,7 @@ (comment (require '[clojure.pprint :as pprint]) (pprint/pprint (models-dev)) - (pprint/pprint (all)) + (pprint/pprint (all (models-dev))) (require '[eca.db :as db]) (sync-models! db/db* (config/all @db/db*) diff --git a/test/eca/models_test.clj b/test/eca/models_test.clj index 1a606a860..b76d20c34 100644 --- a/test/eca/models_test.clj +++ b/test/eca/models_test.clj @@ -102,11 +102,11 @@ (testing "Returns false when provider does not declare API type" (is (false? (#'models/add-models-from-models-dev? - "synthetic" - {} - {:providers {"synthetic" {:url "https://api.synthetic.new/v1"}}} - {:by-id {} - :by-url {"https://api.synthetic.new/v1" {"api" "https://api.synthetic.new/v1"}}})))) + "synthetic" + {} + {:providers {"synthetic" {:url "https://api.synthetic.new/v1"}}} + {:by-id {} + :by-url {"https://api.synthetic.new/v1" {"api" "https://api.synthetic.new/v1"}}})))) (testing "Returns false when provider URL does not match any models.dev api" (is (false? (#'models/add-models-from-models-dev? @@ -141,8 +141,7 @@ {:providers {"my-provider" {:url "https://api.not-matching.test/v1"}}} {:by-id {"my-provider" {"api" "https://api.my-provider.dev/v1" "models" {"foo" {"id" "foo"}}}} - :by-url {}})))) - ) + :by-url {}}))))) (deftest parse-models-dev-provider-models-test (testing "Uses key as model key" @@ -154,7 +153,7 @@ (#'models/parse-models-dev-provider-models "test-provider" {"claude-sonnet-4-5" {"name" "Claude Sonnet 4.5" - "id" "claude-sonnet-4-5"} + "id" "claude-sonnet-4-5"} "gemini-3-flash-preview" {"id" "gemini-3-flash-preview" "name" "Gemini 3 Flash"} "gpt-5.2" {"name" "GPT 5.2" @@ -187,15 +186,15 @@ "bad-entry" 42}))) (is (= 1 (count @warnings*))))))) -(deftest fetch-models-dev-provider-models-test - (with-redefs-fn {#'eca.models/models-dev (constantly {"openai" {"api" "https://api.openai.com" - "models" {"gpt-5.2" {"id" "gpt-5.2" - "name" "GPT 5.2"} - "gpt-4-legacy" {"id" "gpt-4-legacy" - "status" "deprecated"}}} - "anthropic" {"models" {"claude-sonnet-4-5" - {"id" "claude-sonnet-4-5" - "name" "Claude Sonnet 4.5"}}} +(deftest fetch-provider-models-with-priority-models-dev-test + (with-redefs-fn {#'eca.models/models-dev (constantly {"oai-like" {"api" "https://api.openai.com" + "models" {"gpt-5.2" {"id" "gpt-5.2" + "name" "GPT 5.2"} + "gpt-4-legacy" {"id" "gpt-4-legacy" + "status" "deprecated"}}} + "anthropic-like" {"models" {"claude-sonnet-4-5" + {"id" "claude-sonnet-4-5" + "name" "Claude Sonnet 4.5"}}} "synthetic" {"api" "https://api.synthetic.new/v1" "models" {"hf:Qwen/Qwen3-235B-A22B-Instruct-2507" {"id" "hf:Qwen/Qwen3-235B-A22B-Instruct-2507" @@ -203,37 +202,120 @@ "my-provider" {"api" "https://api.my-provider.dev/v1" "models" {"foo" {"id" "foo" "name" "Foo"}}}})} (fn [] - (testing "Loads models for providers with matching URL and API declared" + (testing "Loads models from models.dev when native endpoint is unavailable" (is (match? - {"openai" {"gpt-5.2" {}} - "anthropic" {"claude-sonnet-4-5" {}} + {"oai-like" {"gpt-5.2" {}} + "anthropic-like" {"claude-sonnet-4-5" {}} "synthetic" {"hf:Qwen/Qwen3-235B-A22B-Instruct-2507" {}}} - (#'models/fetch-models-dev-provider-models - {:providers {"openai" {:api "openai-responses" - :url "https://api.openai.com"} - "anthropic" {:api "anthropic" - :url "https://api.anthropic.com"} + (#'models/fetch-provider-models-with-priority + {:providers {"oai-like" {:api "openai-responses" + :url "https://api.openai.com"} + "anthropic-like" {:api "anthropic" + :url "https://api.anthropic.com"} "synthetic" {:api "openai-chat" :url "https://api.synthetic.new/v1"} "my-provider" {:api "openai-chat"} "unknown-url" {:api "openai-chat" - :url "https://api.unknown.test/v1"}}})))) + :url "https://api.unknown.test/v1"}}} + {})))) (testing "Skips models.dev loading when fetchModels is false" (is (match? {} - (#'models/fetch-models-dev-provider-models + (#'models/fetch-provider-models-with-priority {:providers {"synthetic" {:api "openai-chat" :url "https://api.synthetic.new/v1" - :fetchModels false}}})))) + :fetchModels false}}} + {}))))))) + +(deftest fetch-provider-models-with-priority-fetchmodels-disabled-test + (let [native-calls* (atom 0) + models-dev-data {"native-provider" {"api" "https://api.openai.com" + "models" {"from-models-dev" {"id" "from-models-dev"}}}} + config {:providers {"native-provider" {:api "openai-responses" + :url "https://api.openai.com" + :key "sk-test" + :fetchModels false}}}] + (with-redefs [http/get (fn [_url _opts] + (swap! native-calls* inc) + {:status 200 + :body {:data [{:id "native-1"}]}})] + (is (match? + {} + (#'models/fetch-provider-models-with-priority config {} models-dev-data))) + (is (= 0 @native-calls*))))) - (testing "Logs when provider-id fallback is used" - (let [info* (atom [])] - (with-redefs [logger/info (fn [& args] (swap! info* conj args))] - (#'models/fetch-models-dev-provider-models - {:providers {"anthropic" {:api "anthropic" - :url "https://api.anthropic.com"}}}) - (is (some #(re-find #"provider-id fallback" (apply str %)) @info*)))))))) +(deftest fetch-provider-models-with-priority-native-first-test + (let [request* (atom nil) + models-dev-data {"native-provider" {"api" "https://api.openai.com" + "models" {"from-models-dev" {"id" "from-models-dev"}}}}] + (with-redefs [http/get (fn [url opts] + (reset! request* [url opts]) + {:status 200 + :body {:data [{:id "native-1"}]}})] + (is (match? + {"native-provider" {"native-1" {}}} + (#'models/fetch-provider-models-with-priority + {:providers {"native-provider" {:api "openai-responses" + :url "https://api.openai.com" + :key "sk-test"}}} + {} + models-dev-data))) + (is (= "https://api.openai.com/v1/models" (first @request*))) + (is (= "Bearer sk-test" (get-in @request* [1 :headers "Authorization"])))))) + +(deftest fetch-provider-models-with-priority-fallback-test + (let [models-dev-data {"native-provider" {"api" "https://api.openai.com" + "models" {"from-models-dev" {"id" "from-models-dev"}}}}] + (with-redefs [http/get (fn [_url _opts] + {:status 503 + :body {:error "temporary"}})] + (is (match? + {"native-provider" {"from-models-dev" {}}} + (#'models/fetch-provider-models-with-priority + {:providers {"native-provider" {:api "openai-responses" + :url "https://api.openai.com" + :key "sk-test"}}} + {} + models-dev-data)))))) + +(deftest fetch-provider-models-with-priority-invalid-native-payload-test + (let [warnings* (atom []) + models-dev-data {"native-provider" {"api" "https://api.openai.com" + "models" {"from-models-dev" {"id" "from-models-dev"}}}} + config {:providers {"native-provider" {:api "openai-responses" + :url "https://api.openai.com" + :key "sk-test"}}}] + (with-redefs [http/get (fn [_url _opts] + {:status 200 + :body {:models [{:id "native-ignored"}]}}) + logger/warn (fn [& args] (swap! warnings* conj args) nil)] + (is (match? + {"native-provider" {"from-models-dev" {}}} + (#'models/fetch-provider-models-with-priority config {} models-dev-data))) + (is (some #(re-find #"missing sequential :data" (apply str %)) @warnings*))))) + +(deftest fetch-provider-models-with-priority-retry-after-transient-failure-test + (let [native-calls* (atom 0) + models-dev-data {"native-provider" {"api" "https://api.openai.com" + "models" {"from-models-dev" {"id" "from-models-dev"}}}} + config {:providers {"native-provider" {:api "openai-responses" + :url "https://api.openai.com" + :key "sk-test"}}}] + (with-redefs [http/get (fn [_url _opts] + (let [call (swap! native-calls* inc)] + (if (= call 1) + {:status 503 + :body {:error "temporary"}} + {:status 200 + :body {:data [{:id "native-2"}]}})))] + (is (match? + {"native-provider" {"from-models-dev" {}}} + (#'models/fetch-provider-models-with-priority config {} models-dev-data))) + (is (match? + {"native-provider" {"native-2" {}}} + (#'models/fetch-provider-models-with-priority config {} models-dev-data))) + (is (= 2 @native-calls*))))) (deftest build-model-capabilities-test (testing "Uses model-name from config when present"