improves interoperability of model providers, local and remote providers can be used together seemlessly

2025-09-08 22:56:46 +00:00 · 2025-06-09 13:54:37 -04:00
parent ad7dc5c0a6
commit f29bb6779c
14 changed files with 526 additions and 234 deletions
--- a/packages/server/providers/_ProviderRepository.ts
+++ b/packages/server/providers/_ProviderRepository.ts
@@ -0,0 +1,76 @@
+
+export class ProviderRepository {
+    #providers: {name: string, key: string, endpoint: string}[] = [];
+    constructor(env: Record<string, any>) {
+        this.setProviders(env);
+    }
+
+    static OPENAI_COMPAT_ENDPOINTS = {
+        xai: 'https://api.x.ai/v1',
+        groq: 'https://api.groq.com/openai/v1',
+        google: 'https://generativelanguage.googleapis.com/v1beta/openai',
+        fireworks: 'https://api.fireworks.ai/inference/v1',
+        cohere: 'https://api.cohere.ai/compatibility/v1',
+        cloudflare: 'https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1',
+        anthropic: 'https://api.anthropic.com/v1/',
+        openai: 'https://api.openai.com/v1/',
+        cerebras: 'https://api.cerebras.com/v1/',
+        ollama: "http://localhost:11434",
+        mlx: "http://localhost:10240",
+    }
+
+    static async getModelFamily(model, env: Env) {
+        const allModels = await env.KV_STORAGE.get("supportedModels");
+        const models = JSON.parse(allModels);
+        const modelData = models.filter(m => m.id === model)
+        console.log({modelData})
+        return modelData[0].provider;
+    }
+
+    static async getModelMeta(meta, env) {
+        const allModels = await env.KV_STORAGE.get("supportedModels");
+        const models = JSON.parse(allModels);
+        return models.filter(m => m.id === meta.model).pop()
+    }
+
+    getProviders():  {name: string, key: string, endpoint: string}[] {
+        return this.#providers;
+    }
+
+    setProviders(env: Record<string, any>) {
+        let envKeys = Object.keys(env);
+        for (let i = 0; i < envKeys.length; i++) {
+            if (envKeys[i].endsWith('KEY')) {
+                const detectedProvider = envKeys[i].split('_')[0].toLowerCase();
+                switch (detectedProvider) {
+                    case 'anthropic':
+                        this.#providers.push({
+                            name: 'anthropic',
+                            key: env.ANTHROPIC_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS['anthropic']
+                        });
+                        break;
+                    case 'gemini':
+                        this.#providers.push({
+                            name: 'google',
+                            key: env.GEMINI_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS['google']
+                        });
+                        break;
+                    case 'cloudflare':
+                        this.#providers.push({
+                            name: 'cloudflare',
+                            key: env.CLOUDFLARE_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID)
+                        })
+                    default:
+                        this.#providers.push({
+                            name: detectedProvider,
+                            key: env[envKeys[i]],
+                            endpoint: OPENAI_COMPAT_ENDPOINTS[detectedProvider]
+                        });
+                }
+            }
+        }
+    }
+}