improves interoperability of model providers, local and remote providers can be used together seemlessly

packages/client/src/components/chat/input-menu/InputMenu.tsx

@@ -1,215 +1,197 @@
-import React, { useCallback, useEffect, useRef, useState } from "react";
+import React, {useCallback, useEffect, useRef, useState} from "react";
 import {
-  Box,
+    Box,
-  Button,
+    Button,
-  Divider,
+    Divider,
-  Flex,
+    Flex,
-  IconButton,
+    IconButton,
-  Menu,
+    Menu,
-  MenuButton,
+    MenuButton,
-  MenuItem,
+    MenuItem,
-  MenuList,
+    MenuList,
-  Text,
+    Text,
-  useDisclosure,
+    useDisclosure,
-  useOutsideClick,
+    useOutsideClick,
 } from "@chakra-ui/react";
-import { observer } from "mobx-react-lite";
+import {observer} from "mobx-react-lite";
-import { ChevronDown, Copy, RefreshCcw, Settings } from "lucide-react";
+import {ChevronDown, Copy, RefreshCcw, Settings} from "lucide-react";
-import ClientChatStore from "../../../stores/ClientChatStore";
 import clientChatStore from "../../../stores/ClientChatStore";
 import FlyoutSubMenu from "./FlyoutSubMenu";
-import { useIsMobile } from "../../contexts/MobileContext";
+import {useIsMobile} from "../../contexts/MobileContext";
-import { useIsMobile as useIsMobileUserAgent } from "../../../hooks/_IsMobileHook";
+import {useIsMobile as useIsMobileUserAgent} from "../../../hooks/_IsMobileHook";
-import { getModelFamily, SUPPORTED_MODELS } from "../lib/SupportedModels";
+import {formatConversationMarkdown} from "../lib/exportConversationAsMarkdown";
-import { formatConversationMarkdown } from "../lib/exportConversationAsMarkdown";
 
 export const MsM_commonButtonStyles = {
-  bg: "transparent",
+    bg: "transparent",
-  color: "text.primary",
+    color: "text.primary",
-  borderRadius: "full",
+    borderRadius: "full",
-  padding: 2,
+    padding: 2,
-  border: "none",
+    border: "none",
-  _hover: { bg: "rgba(255, 255, 255, 0.2)" },
+    _hover: {bg: "rgba(255, 255, 255, 0.2)"},
-  _active: { bg: "rgba(255, 255, 255, 0.3)" },
+    _active: {bg: "rgba(255, 255, 255, 0.3)"},
-  _focus: { boxShadow: "none" },
+    _focus: {boxShadow: "none"},
 };
 
 const InputMenu: React.FC<{ isDisabled?: boolean }> = observer(
-  ({ isDisabled }) => {
+    ({isDisabled}) => {
-    const isMobile = useIsMobile();
+        const isMobile = useIsMobile();
-    const isMobileUserAgent = useIsMobileUserAgent();
+        const isMobileUserAgent = useIsMobileUserAgent();
-    const {
+        const {
-      isOpen,
+            isOpen,
-      onOpen,
+            onOpen,
-      onClose,
+            onClose,
-      onToggle,
+            onToggle,
-      getDisclosureProps,
+            getDisclosureProps,
-      getButtonProps,
+            getButtonProps,
-    } = useDisclosure();
+        } = useDisclosure();
 
-    const [controlledOpen, setControlledOpen] = useState<boolean>(false);
+        const [controlledOpen, setControlledOpen] = useState<boolean>(false);
+        const [supportedModels, setSupportedModels] = useState<any[]>([]);
 
-    useEffect(() => {
+        useEffect(() => {
-      setControlledOpen(isOpen);
+            setControlledOpen(isOpen);
-    }, [isOpen]);
+        }, [isOpen]);
+
+        useEffect(() => {
+            fetch("/api/models").then(response => response.json()).then((models) => {
+                setSupportedModels(models);
+            }).catch((err) => {
+                console.error("Could not fetch models: ", err);
+            });
+        }, []);
 
 
-    const getSupportedModels = async () => {
+        const handleClose = useCallback(() => {
-        // Check if fetch is available (browser environment)
+            onClose();
-        if (typeof fetch !== 'undefined') {
+        }, [isOpen]);
-            try {
+
-                return await (await fetch("/api/models")).json();
+        const handleCopyConversation = useCallback(() => {
-            } catch (error) {
+            navigator.clipboard
-                console.error("Error fetching models:", error);
+                .writeText(formatConversationMarkdown(clientChatStore.items))
-                return [];
+                .then(() => {
-            }
+                    window.alert(
-        } else {
+                        "Conversation copied to clipboard. \n\nPaste it somewhere safe!",
-            // In test environment or where fetch is not available
+                    );
-            console.log("Fetch not available, using default models");
+                    onClose();
-            return [];
+                })
+                .catch((err) => {
+                    console.error("Could not copy text to clipboard: ", err);
+                    window.alert("Failed to copy conversation. Please try again.");
+                });
+        }, [onClose]);
+
+        async function selectModelFn({name, value}) {
+            clientChatStore.setModel(value);
         }
-    }
 
-      useEffect(() => {
+        function isSelectedModelFn({name, value}) {
-           getSupportedModels().then((supportedModels) => {
+            return clientChatStore.model === value;
-               // Check if setSupportedModels method exists before calling it
+        }
-               if (clientChatStore.setSupportedModels) {
-                   clientChatStore.setSupportedModels(supportedModels);
-               } else {
-                   console.log("setSupportedModels method not available in this environment");
-               }
-           });
-      }, []);
 
+        const menuRef = useRef();
+        const [menuState, setMenuState] = useState();
 
-    const handleClose = useCallback(() => {
+        useOutsideClick({
-      onClose();
+            enabled: !isMobile && isOpen,
-    }, [isOpen]);
+            ref: menuRef,
-
+            handler: () => {
-    const handleCopyConversation = useCallback(() => {
+                handleClose();
-      navigator.clipboard
+            },
-        .writeText(formatConversationMarkdown(clientChatStore.items))
-        .then(() => {
-          window.alert(
-            "Conversation copied to clipboard. \n\nPaste it somewhere safe!",
-          );
-          onClose();
-        })
-        .catch((err) => {
-          console.error("Could not copy text to clipboard: ", err);
-          window.alert("Failed to copy conversation. Please try again.");
         });
-    }, [onClose]);
 
-    async function selectModelFn({ name, value }) {
+        return (
-        clientChatStore.setModel(value);
+            <Menu
-    }
+                isOpen={controlledOpen}
-
+                onClose={onClose}
-    function isSelectedModelFn({ name, value }) {
+                onOpen={onOpen}
-      return clientChatStore.model === value;
+                autoSelect={false}
-    }
+                closeOnSelect={false}
-
+                closeOnBlur={isOpen && !isMobileUserAgent}
-    const menuRef = useRef();
+                isLazy={true}
-    const [menuState, setMenuState] = useState();
+                lazyBehavior={"unmount"}
-
+            >
-    useOutsideClick({
+                {isMobile ? (
-      enabled: !isMobile && isOpen,
+                    <MenuButton
-      ref: menuRef,
+                        as={IconButton}
-      handler: () => {
+                        bg="text.accent"
-        handleClose();
+                        icon={<Settings size={20}/>}
-      },
+                        isDisabled={isDisabled}
-    });
+                        aria-label="Settings"
-
+                        _hover={{bg: "rgba(255, 255, 255, 0.2)"}}
-    return (
+                        _focus={{boxShadow: "none"}}
-      <Menu
+                        {...MsM_commonButtonStyles}
-        isOpen={controlledOpen}
+                    />
-        onClose={onClose}
+                ) : (
-        onOpen={onOpen}
+                    <MenuButton
-        autoSelect={false}
+                        as={Button}
-        closeOnSelect={false}
+                        rightIcon={<ChevronDown size={16}/>}
-        closeOnBlur={isOpen && !isMobileUserAgent}
+                        isDisabled={isDisabled}
-        isLazy={true}
+                        variant="ghost"
-        lazyBehavior={"unmount"}
+                        display="flex"
-      >
+                        justifyContent="space-between"
-        {isMobile ? (
+                        alignItems="center"
-          <MenuButton
+                        minW="auto"
-            as={IconButton}
+                        {...MsM_commonButtonStyles}
-            bg="text.accent"
+                    >
-            icon={<Settings size={20} />}
+                        <Text noOfLines={1} maxW="100px" fontSize="sm">
-            isDisabled={isDisabled}
+                            {clientChatStore.model}
-            aria-label="Settings"
+                        </Text>
-            _hover={{ bg: "rgba(255, 255, 255, 0.2)" }}
+                    </MenuButton>
-            _focus={{ boxShadow: "none" }}
+                )}
-            {...MsM_commonButtonStyles}
+                <MenuList
-          />
+                    bg="background.tertiary"
-        ) : (
+                    border="none"
-          <MenuButton
+                    borderRadius="md"
-            as={Button}
+                    boxShadow="lg"
-            rightIcon={<ChevronDown size={16} />}
+                    minW={"10rem"}
-            isDisabled={isDisabled}
+                    ref={menuRef}
-            variant="ghost"
+                >
-            display="flex"
+                    <FlyoutSubMenu
-            justifyContent="space-between"
+                        title="Text Models"
-            alignItems="center"
+                        flyoutMenuOptions={supportedModels.map((modelData) => ({
-            minW="auto"
+                            name: modelData.id.split('/').pop() || modelData.id,
-            {...MsM_commonButtonStyles}
+                            value: modelData.id
-          >
+                        }))}
-            <Text noOfLines={1} maxW="100px" fontSize="sm">
+                        onClose={onClose}
-              {clientChatStore.model}
+                        parentIsOpen={isOpen}
-            </Text>
+                        setMenuState={setMenuState}
-          </MenuButton>
+                        handleSelect={selectModelFn}
-        )}
+                        isSelected={isSelectedModelFn}
-        <MenuList
+                    />
-          bg="background.tertiary"
+                    <Divider color="text.tertiary"/>
-          border="none"
+                    {/*Export conversation button*/}
-          borderRadius="md"
+                    <MenuItem
-          boxShadow="lg"
+                        bg="background.tertiary"
-          minW={"10rem"}
+                        color="text.primary"
-          ref={menuRef}
+                        onClick={handleCopyConversation}
-        >
+                        _hover={{bg: "rgba(0, 0, 0, 0.05)"}}
-          <FlyoutSubMenu
+                        _focus={{bg: "rgba(0, 0, 0, 0.1)"}}
-            title="Text Models"
+                    >
-            flyoutMenuOptions={clientChatStore.supportedModels.map((m) => ({ name: m, value: m }))}
+                        <Flex align="center">
-            onClose={onClose}
+                            <Copy size="16px" style={{marginRight: "8px"}}/>
-            parentIsOpen={isOpen}
+                            <Box>Export</Box>
-            setMenuState={setMenuState}
+                        </Flex>
-            handleSelect={selectModelFn}
+                    </MenuItem>
-            isSelected={isSelectedModelFn}
+                    {/*New conversation button*/}
-          />
+                    <MenuItem
-          <Divider color="text.tertiary" />
+                        bg="background.tertiary"
-          {/*Export conversation button*/}
+                        color="text.primary"
-          <MenuItem
+                        onClick={() => {
-            bg="background.tertiary"
+                            clientChatStore.setActiveConversation("conversation:new");
-            color="text.primary"
+                            onClose();
-            onClick={handleCopyConversation}
+                        }}
-            _hover={{ bg: "rgba(0, 0, 0, 0.05)" }}
+                        _hover={{bg: "rgba(0, 0, 0, 0.05)"}}
-            _focus={{ bg: "rgba(0, 0, 0, 0.1)" }}
+                        _focus={{bg: "rgba(0, 0, 0, 0.1)"}}
-          >
+                    >
-            <Flex align="center">
+                        <Flex align="center">
-              <Copy size="16px" style={{ marginRight: "8px" }} />
+                            <RefreshCcw size="16px" style={{marginRight: "8px"}}/>
-              <Box>Export</Box>
+                            <Box>New</Box>
-            </Flex>
+                        </Flex>
-          </MenuItem>
+                    </MenuItem>
-          {/*New conversation button*/}
+                </MenuList>
-          <MenuItem
+            </Menu>
-            bg="background.tertiary"
+        );
-            color="text.primary"
+    },
-            onClick={() => {
-              clientChatStore.setActiveConversation("conversation:new");
-              onClose();
-            }}
-            _hover={{ bg: "rgba(0, 0, 0, 0.05)" }}
-            _focus={{ bg: "rgba(0, 0, 0, 0.1)" }}
-          >
-            <Flex align="center">
-              <RefreshCcw size="16px" style={{ marginRight: "8px" }} />
-              <Box>New</Box>
-            </Flex>
-          </MenuItem>
-        </MenuList>
-      </Menu>
-    );
-  },
 );
 
 export default InputMenu;

packages/server/ServerCoordinator.ts

@@ -1,6 +1,9 @@
 import { DurableObject } from "cloudflare:workers";
+import {ProviderRepository} from "./providers/_ProviderRepository";
 
 export default class ServerCoordinator extends DurableObject {
+  env;
+  state;
   constructor(state, env) {
     super(state, env);
     this.state = state;
@@ -8,20 +11,24 @@ export default class ServerCoordinator extends DurableObject {
   }
 
   // Public method to calculate dynamic max tokens
-  async dynamicMaxTokens(input, maxOuputTokens) {
+  async dynamicMaxTokens(model, input, maxOuputTokens) {
-    return 2000;
+
-    // const baseTokenLimit = 1024;
+    const modelMeta = ProviderRepository.getModelMeta(model, this.env);
-    //
+
-    //
+    // The token‑limit information is stored in three different keys:
-    // const { encode } = await import("gpt-tokenizer/esm/model/gpt-4o");
+    // max_completion_tokens
-    //
+    // context_window
-    // const inputTokens = Array.isArray(input)
+    // context_length
-    //     ? encode(input.map(i => i.content).join(' '))
+
-    //     : encode(input);
+    if('max_completion_tokens' in modelMeta) {
-    //
+      return modelMeta.max_completion_tokens;
-    // const scalingFactor = inputTokens.length > 300 ? 1.5 : 1;
+    } else if('context_window' in modelMeta) {
-    //
+      return modelMeta.context_window;
-    // return Math.min(baseTokenLimit + Math.floor(inputTokens.length * scalingFactor^2), maxOuputTokens);
+    } else if('context_length' in modelMeta) {
+      return modelMeta.context_length;
+    } else {
+      return 8096;
+    }
   }
 
   // Public method to retrieve conversation history

packages/server/lib/chat-sdk.ts

@@ -1,8 +1,8 @@
 import {OpenAI} from "openai";
 import Message from "../models/Message.ts";
 import {AssistantSdk} from "./assistant-sdk.ts";
-import {getModelFamily} from "@open-gsio/ai/supported-models.ts";
 import type {Instance} from "mobx-state-tree";
+import {ProviderRepository} from "../providers/_ProviderRepository";
 
 export class ChatSdk {
     static async preprocess({
@@ -95,9 +95,10 @@ export class ChatSdk {
             assistantPrompt: string;
             toolResults: Instance<typeof Message>;
             model: any;
+            env: Env;
         },
     ) {
-        const modelFamily = getModelFamily(opts.model);
+        const modelFamily = ProviderRepository.getModelFamily(opts.model, opts.env)
 
         const messagesToSend = [];
 

packages/server/providers/_ProviderRepository.ts

@@ -0,0 +1,76 @@
+
+export class ProviderRepository {
+    #providers: {name: string, key: string, endpoint: string}[] = [];
+    constructor(env: Record<string, any>) {
+        this.setProviders(env);
+    }
+
+    static OPENAI_COMPAT_ENDPOINTS = {
+        xai: 'https://api.x.ai/v1',
+        groq: 'https://api.groq.com/openai/v1',
+        google: 'https://generativelanguage.googleapis.com/v1beta/openai',
+        fireworks: 'https://api.fireworks.ai/inference/v1',
+        cohere: 'https://api.cohere.ai/compatibility/v1',
+        cloudflare: 'https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1',
+        anthropic: 'https://api.anthropic.com/v1/',
+        openai: 'https://api.openai.com/v1/',
+        cerebras: 'https://api.cerebras.com/v1/',
+        ollama: "http://localhost:11434",
+        mlx: "http://localhost:10240",
+    }
+
+    static async getModelFamily(model, env: Env) {
+        const allModels = await env.KV_STORAGE.get("supportedModels");
+        const models = JSON.parse(allModels);
+        const modelData = models.filter(m => m.id === model)
+        console.log({modelData})
+        return modelData[0].provider;
+    }
+
+    static async getModelMeta(meta, env) {
+        const allModels = await env.KV_STORAGE.get("supportedModels");
+        const models = JSON.parse(allModels);
+        return models.filter(m => m.id === meta.model).pop()
+    }
+
+    getProviders():  {name: string, key: string, endpoint: string}[] {
+        return this.#providers;
+    }
+
+    setProviders(env: Record<string, any>) {
+        let envKeys = Object.keys(env);
+        for (let i = 0; i < envKeys.length; i++) {
+            if (envKeys[i].endsWith('KEY')) {
+                const detectedProvider = envKeys[i].split('_')[0].toLowerCase();
+                switch (detectedProvider) {
+                    case 'anthropic':
+                        this.#providers.push({
+                            name: 'anthropic',
+                            key: env.ANTHROPIC_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS['anthropic']
+                        });
+                        break;
+                    case 'gemini':
+                        this.#providers.push({
+                            name: 'google',
+                            key: env.GEMINI_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS['google']
+                        });
+                        break;
+                    case 'cloudflare':
+                        this.#providers.push({
+                            name: 'cloudflare',
+                            key: env.CLOUDFLARE_API_KEY,
+                            endpoint: OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID)
+                        })
+                    default:
+                        this.#providers.push({
+                            name: detectedProvider,
+                            key: env[envKeys[i]],
+                            endpoint: OPENAI_COMPAT_ENDPOINTS[detectedProvider]
+                        });
+                }
+            }
+        }
+    }
+}

packages/server/providers/cerebras.ts

@@ -1,10 +1,11 @@
 import {OpenAI} from "openai";
 import {BaseChatProvider, CommonProviderParams} from "./chat-stream-provider.ts";
+import {ProviderRepository} from "./_ProviderRepository";
 
 export class CerebrasChatProvider extends BaseChatProvider {
   getOpenAIClient(param: CommonProviderParams): OpenAI {
     return new OpenAI({
-      baseURL: "https://api.cerebras.ai/v1",
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.cerebras,
       apiKey: param.env.CEREBRAS_API_KEY,
     });
   }

packages/server/providers/chat-stream-provider.ts

@@ -35,6 +35,7 @@ export abstract class BaseChatProvider implements ChatStreamProvider {
       model: param.model,
       assistantPrompt,
       toolResults: param.preprocessedContext,
+      env: param.env
     });
 
     const client = this.getOpenAIClient(param);

packages/server/providers/claude.ts

@@ -69,6 +69,7 @@ export class ClaudeChatProvider extends BaseChatProvider {
       model: param.model,
       assistantPrompt,
       toolResults: param.preprocessedContext,
+      env: param.env,
     });
 
     const streamParams = this.getStreamParams(param, safeMessages);

packages/server/providers/cloudflareAi.ts

@@ -1,13 +1,12 @@
 import {OpenAI} from "openai";
 import {BaseChatProvider, CommonProviderParams} from "./chat-stream-provider.ts";
+import {ProviderRepository} from "./_ProviderRepository";
 
 export class CloudflareAiChatProvider extends BaseChatProvider {
   getOpenAIClient(param: CommonProviderParams): OpenAI {
-    const cfAiURL = `https://api.cloudflare.com/client/v4/accounts/${param.env.CLOUDFLARE_ACCOUNT_ID}/ai/v1`;
-
     return new OpenAI({
       apiKey: param.env.CLOUDFLARE_API_KEY,
-      baseURL: cfAiURL,
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.cloudflare.replace("{CLOUDFLARE_ACCOUNT_ID}", param.env.CLOUDFLARE_ACCOUNT_ID),
     });
   }
 

packages/server/providers/fireworks.ts

@@ -11,12 +11,13 @@ import {
 import Message from "../models/Message.ts";
 import ChatSdk from "../lib/chat-sdk.ts";
 import { BaseChatProvider, CommonProviderParams } from "./chat-stream-provider.ts";
+import {ProviderRepository} from "./_ProviderRepository";
 
 export class FireworksAiChatProvider extends BaseChatProvider {
   getOpenAIClient(param: CommonProviderParams): OpenAI {
     return new OpenAI({
       apiKey: param.env.FIREWORKS_API_KEY,
-      baseURL: "https://api.fireworks.ai/inference/v1",
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.fireworks,
     });
   }
 

packages/server/providers/google.ts

@@ -2,11 +2,12 @@ import { OpenAI } from "openai";
 import ChatSdk from "../lib/chat-sdk.ts";
 import { StreamParams } from "../services/ChatService.ts";
 import { BaseChatProvider, CommonProviderParams } from "./chat-stream-provider.ts";
+import {ProviderRepository} from "./_ProviderRepository";
 
 export class GoogleChatProvider extends BaseChatProvider {
   getOpenAIClient(param: CommonProviderParams): OpenAI {
     return new OpenAI({
-      baseURL: "https://generativelanguage.googleapis.com/v1beta/openai",
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.google,
       apiKey: param.env.GEMINI_API_KEY,
     });
   }

packages/server/providers/groq.ts

@@ -7,11 +7,12 @@ import {
   UnionStringArray,
 } from "mobx-state-tree";
 import { BaseChatProvider, CommonProviderParams } from "./chat-stream-provider.ts";
+import {ProviderRepository} from "./_ProviderRepository";
 
 export class GroqChatProvider extends BaseChatProvider {
   getOpenAIClient(param: CommonProviderParams): OpenAI {
     return new OpenAI({
-      baseURL: "https://api.groq.com/openai/v1",
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.groq,
       apiKey: param.env.GROQ_API_KEY,
     });
   }

packages/server/providers/mlx-omni.ts

@@ -0,0 +1,73 @@
+import { OpenAI } from "openai";
+import { BaseChatProvider, CommonProviderParams } from "./chat-stream-provider.ts";
+
+export class MlxOmniChatProvider extends BaseChatProvider {
+    getOpenAIClient(param: CommonProviderParams): OpenAI {
+        return new OpenAI({
+            baseURL: param.env.MLX_API_ENDPOINT ?? "http://localhost:10240",
+            apiKey: param.env.MLX_API_KEY,
+        });
+    }
+
+    getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+        const tuningParams = {
+            temperature: 0.75,
+        };
+
+        const getTuningParams = () => {
+            return tuningParams;
+        };
+
+        return {
+            model: param.model,
+            messages: safeMessages,
+            stream: true,
+            ...getTuningParams(),
+        };
+    }
+
+    async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+        if (chunk.choices && chunk.choices[0]?.finish_reason === "stop") {
+            dataCallback({ type: "chat", data: chunk });
+            return true;
+        }
+
+        dataCallback({ type: "chat", data: chunk });
+        return false;
+    }
+}
+
+export class MlxOmniChatSdk {
+    private static provider = new MlxOmniChatProvider();
+
+    static async handleMlxOmniStream(
+        ctx: {
+            openai: OpenAI;
+            systemPrompt: any;
+            preprocessedContext: any;
+            maxTokens: unknown | number | undefined;
+            messages: any;
+            disableWebhookGeneration: boolean;
+            model: any;
+            env: Env;
+        },
+        dataCallback: (data: any) => any,
+    ) {
+        if (!ctx.messages?.length) {
+            return new Response("No messages provided", { status: 400 });
+        }
+
+        return this.provider.handleStream(
+            {
+                systemPrompt: ctx.systemPrompt,
+                preprocessedContext: ctx.preprocessedContext,
+                maxTokens: ctx.maxTokens,
+                messages: ctx.messages,
+                model: ctx.model,
+                env: ctx.env,
+                disableWebhookGeneration: ctx.disableWebhookGeneration,
+            },
+            dataCallback,
+        );
+    }
+}

packages/server/providers/ollama.ts

@@ -0,0 +1,73 @@
+import { OpenAI } from "openai";
+import { BaseChatProvider, CommonProviderParams } from "./chat-stream-provider.ts";
+
+export class OllamaChatProvider extends BaseChatProvider {
+    getOpenAIClient(param: CommonProviderParams): OpenAI {
+        return new OpenAI({
+            baseURL: param.env.OLLAMA_API_ENDPOINT ?? ,
+            apiKey: param.env.OLLAMA_API_KEY,
+        });
+    }
+
+    getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+        const tuningParams = {
+            temperature: 0.75,
+        };
+
+        const getTuningParams = () => {
+            return tuningParams;
+        };
+
+        return {
+            model: param.model,
+            messages: safeMessages,
+            stream: true,
+            ...getTuningParams(),
+        };
+    }
+
+    async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+        if (chunk.choices && chunk.choices[0]?.finish_reason === "stop") {
+            dataCallback({ type: "chat", data: chunk });
+            return true;
+        }
+
+        dataCallback({ type: "chat", data: chunk });
+        return false;
+    }
+}
+
+export class OllamaChatSdk {
+    private static provider = new OllamaChatProvider();
+
+    static async handleOllamaStream(
+        ctx: {
+            openai: OpenAI;
+            systemPrompt: any;
+            preprocessedContext: any;
+            maxTokens: unknown | number | undefined;
+            messages: any;
+            disableWebhookGeneration: boolean;
+            model: any;
+            env: Env;
+        },
+        dataCallback: (data: any) => any,
+    ) {
+        if (!ctx.messages?.length) {
+            return new Response("No messages provided", { status: 400 });
+        }
+
+        return this.provider.handleStream(
+            {
+                systemPrompt: ctx.systemPrompt,
+                preprocessedContext: ctx.preprocessedContext,
+                maxTokens: ctx.maxTokens,
+                messages: ctx.messages,
+                model: ctx.model,
+                env: ctx.env,
+                disableWebhookGeneration: ctx.disableWebhookGeneration,
+            },
+            dataCallback,
+        );
+    }
+}

packages/server/services/ChatService.ts

@@ -3,7 +3,6 @@ import OpenAI from 'openai';
 import ChatSdk from '../lib/chat-sdk.ts';
 import Message from "../models/Message.ts";
 import O1Message from "../models/O1Message.ts";
-import {getModelFamily, ModelFamily, SUPPORTED_MODELS} from "@open-gsio/ai/supported-models";
 import {OpenAiChatSdk} from "../providers/openai.ts";
 import {GroqChatSdk} from "../providers/groq.ts";
 import {ClaudeChatSdk} from "../providers/claude.ts";
@@ -13,6 +12,9 @@ import {GoogleChatSdk} from "../providers/google.ts";
 import {XaiChatSdk} from "../providers/xai.ts";
 import {CerebrasSdk} from "../providers/cerebras.ts";
 import {CloudflareAISdk} from "../providers/cloudflareAi.ts";
+import {OllamaChatSdk} from "../providers/ollama";
+import {MlxOmniChatSdk} from "../providers/mlx-omni";
+import {ProviderRepository} from "../providers/_ProviderRepository";
 
 export interface StreamParams {
     env: Env;
@@ -110,24 +112,97 @@ const ChatService = types
             cerebras: (params: StreamParams, dataHandler: Function) =>
                 CerebrasSdk.handleCerebrasStream(params, dataHandler),
             cloudflareAI: (params: StreamParams, dataHandler: Function) =>
-                CloudflareAISdk.handleCloudflareAIStream(params, dataHandler)
+                CloudflareAISdk.handleCloudflareAIStream(params, dataHandler),
+            ollama: (params: StreamParams, dataHandler: Function) =>
+                OllamaChatSdk.handleOllamaStream(params, dataHandler),
+            mlx: (params: StreamParams, dataHandler: Function) =>
+                MlxOmniChatSdk.handleMlxOmniStream(params, dataHandler),
         };
 
         return {
-            async getSupportedModels() {
+            getSupportedModels: flow(function* ():
-                const isLocal = self.env.OPENAI_API_ENDPOINT && self.env.OPENAI_API_ENDPOINT.includes("localhost");
+                Generator<Promise<unknown>, Response, unknown> {
-                console.log({isLocal})
+
-                if(isLocal) {
+                // ----- Helpers ----------------------------------------------------------
-                    console.log("getting local models")
+                const logger = console;
-                    const openaiClient = new OpenAI({baseURL: self.env.OPENAI_API_ENDPOINT})
+
-                    const models = await openaiClient.models.list();
+                // ----- 1. Try cached value ---------------------------------------------
-                    return Response.json(
+                try {
-                        models.data
+                    const cached = yield self.env.KV_STORAGE.get('supportedModels');
-                            .filter(model => model.id.includes("mlx"))
+                    if (cached) {
-                            .map(model => model.id));
+                        const parsed = JSON.parse(cached as string);
+                        if (Array.isArray(parsed)) {
+                            logger.info('Cache hit – returning supportedModels from KV');
+                            return new Response(JSON.stringify(parsed), { status: 200 });
+                        }
+                        logger.warn('Cache entry malformed – refreshing');
+                    }
+                } catch (err) {
+                    logger.error('Error reading/parsing supportedModels cache', err);
                 }
-                return Response.json(SUPPORTED_MODELS);
+
-            },
+                // ----- 2. Build fresh list ---------------------------------------------
+                const providerRepo   = new ProviderRepository(self.env);
+                const providers      = providerRepo.getProviders();
+                const providerModels = new Map<string, any[]>();
+                const modelMeta      = new Map<string, any>();
+
+                for (const provider of providers) {
+                    if (!provider.key) continue;
+
+                    logger.info(`Fetching models for provider «${provider.name}»`);
+
+                    const openai = new OpenAI({ apiKey: provider.key, baseURL: provider.endpoint });
+
+                    // 2‑a. List models
+                    try {
+                        const listResp = yield openai.models.list();          // <‑‑ async
+                        const models   = ('data' in listResp) ? listResp.data : listResp;
+                        providerModels.set(provider.name, models);
+
+                        // 2‑b. Retrieve metadata
+                        for (const mdl of models) {
+                            try {
+                                const meta = yield openai.models.retrieve(mdl.id); // <‑‑ async
+                                modelMeta.set(mdl.id, { ...mdl, ...meta });
+                            } catch (err) {
+                                // logger.error(`Metadata fetch failed for ${mdl.id}`, err);
+                                modelMeta.set(mdl.id, {provider: provider.name, mdl});
+                            }
+                        }
+                    } catch (err) {
+                        logger.error(`Model list failed for provider «${provider.name}»`, err);
+                    }
+                }
+
+                // ----- 3. Merge results -------------------------------------------------
+                const resultMap = new Map<string, any>();
+                for (const [provName, models] of providerModels) {
+                    for (const mdl of models) {
+                        resultMap.set(mdl.id, {
+                            id: mdl.id,
+                            provider: provName,
+                            ...(modelMeta.get(mdl.id) ?? mdl),
+                        });
+                    }
+                }
+                const resultArr = Array.from(resultMap.values());
+
+                // ----- 4. Cache fresh list ---------------------------------------------
+                try {
+                    yield self.env.KV_STORAGE.put(
+                        'supportedModels',
+                        JSON.stringify(resultArr),
+                        { expirationTtl: 60 * 60 * 24 },   // 24 h
+                    );
+                    logger.info('supportedModels cache refreshed');
+                } catch (err) {
+                    logger.error('KV put failed for supportedModels', err);
+                }
+
+                // ----- 5. Return --------------------------------------------------------
+                return new Response(JSON.stringify(resultArr), { status: 200 });
+            }),
             setActiveStream(streamId: string, stream: any) {
                 const validStream = {
                     name: stream?.name || "Unnamed Stream",
@@ -179,13 +254,13 @@ const ChatService = types
                 const {streamConfig, streamParams, controller, encoder, streamId} = params;
 
                 const useModelFamily = () => {
-                    return !self.env.OPENAI_API_ENDPOINT || !self.env.OPENAI_API_ENDPOINT.includes("localhost") ? getModelFamily(streamConfig.model) : "openai";
+                    return ProviderRepository.getModelFamily(streamConfig.model, self.env)
                 }
 
-                const modelFamily = useModelFamily();
+                const modelFamily = await useModelFamily();
 
                 const useModelHandler = () => {
-                    return !self.env.OPENAI_API_ENDPOINT || !self.env.OPENAI_API_ENDPOINT.includes("localhost") ? modelHandlers[modelFamily as ModelFamily] : modelHandlers.openai;
+                    return modelHandlers[modelFamily]
                 }
 
                 const handler = useModelHandler();