diff --git a/packages/client/src/pages/index/+Page.tsx b/packages/client/src/pages/index/+Page.tsx index 07fb121..83fbd37 100644 --- a/packages/client/src/pages/index/+Page.tsx +++ b/packages/client/src/pages/index/+Page.tsx @@ -2,7 +2,6 @@ import React, { useEffect } from "react"; import { Stack } from "@chakra-ui/react"; import Chat from "../../components/chat/Chat"; import clientChatStore from "../../stores/ClientChatStore"; -import { getModelFamily } from "../../components/chat/lib/SupportedModels"; // renders "/" export default function IndexPage() { diff --git a/packages/client/src/stores/UserOptionsStore.ts b/packages/client/src/stores/UserOptionsStore.ts index 756db03..27677bd 100644 --- a/packages/client/src/stores/UserOptionsStore.ts +++ b/packages/client/src/stores/UserOptionsStore.ts @@ -23,7 +23,7 @@ export const UserOptionsStoreModel = types .split(";") .find((row) => row.startsWith("user_preferences")); - console.log(document.cookie.split(";")); + // console.log(document.cookie.split(";")); const newUserOptions = JSON.stringify({ theme: self.theme, @@ -46,7 +46,7 @@ export const UserOptionsStoreModel = types .find((row) => row.startsWith("user_preferences")); if (!userPreferencesCoookie) { - console.log("No user preferences cookie found, creating one"); + // console.log("No user preferences cookie found, creating one"); self.storeUserOptions(); } @@ -60,20 +60,20 @@ export const UserOptionsStoreModel = types window.addEventListener("scroll", () => { if (ClientChatStore.isLoading && self.followModeEnabled) { - console.log("scrolling"); + // console.log("scrolling"); self.setFollowModeEnabled(false); } }); window.addEventListener("wheel", () => { if (ClientChatStore.isLoading && self.followModeEnabled) { - console.log("wheel"); + // console.log("wheel"); self.setFollowModeEnabled(false); } }); window.addEventListener("touchmove", () => { - console.log("touchmove"); + // console.log("touchmove"); if (ClientChatStore.isLoading && self.followModeEnabled) { self.setFollowModeEnabled(false); } diff --git a/packages/env/env.d.ts b/packages/env/env.d.ts index fdb595b..9c2b327 100644 --- a/packages/env/env.d.ts +++ b/packages/env/env.d.ts @@ -12,7 +12,9 @@ interface Env { // KV Bindings KV_STORAGE: KVNamespace; + // Text/Secrets + METRICS_HOST: string; OPENAI_API_ENDPOINT: string; OPENAI_API_KEY: string; EVENTSOURCE_HOST: string; @@ -24,4 +26,6 @@ interface Env { CEREBRAS_API_KEY: string; CLOUDFLARE_API_KEY: string; CLOUDFLARE_ACCOUNT_ID: string; + MLX_API_KEY: string; + OLLAMA_API_KEY: string; } diff --git a/packages/scripts/configure_local_inference.sh b/packages/scripts/configure_local_inference.sh index f4c725d..070a2cd 100755 --- a/packages/scripts/configure_local_inference.sh +++ b/packages/scripts/configure_local_inference.sh @@ -17,11 +17,18 @@ configure_dev_vars() { # Default URL is automatic but can be overridden for remote deployments if [[ "$endpoint_url" == *"11434"* ]]; then + echo "OPENAI_API_ENDPOINT=http://localhost:11434" >> "${ENV_LOCAL_PATH}" echo "OLLAMA_API_KEY=active" >> "${ENV_LOCAL_PATH}" + + echo "OPENAI_API_ENDPOINT=http://localhost:11434" >> "${DEV_VARS_PATH}" echo "OLLAMA_API_KEY=active" >> "${DEV_VARS_PATH}" fi if [[ "$endpoint_url" == *"10240"* ]]; then + + echo "OPENAI_API_ENDPOINT=http://localhost:10240/v1" >> "${ENV_LOCAL_PATH}" echo "MLX_API_KEY=active" >> "${ENV_LOCAL_PATH}" + + echo "OPENAI_API_ENDPOINT=http://localhost:10240/v1" >> "${DEV_VARS_PATH}" echo "MLX_API_KEY=active" >> "${DEV_VARS_PATH}" fi diff --git a/packages/server/api-router.ts b/packages/server/api-router.ts index f3d0955..60e2cde 100644 --- a/packages/server/api-router.ts +++ b/packages/server/api-router.ts @@ -57,14 +57,15 @@ export function createRouter() { // return documentService.handleGetDocument(r) // }) - .all("/api/metrics/*", async (r, e, c) => { + .all("/api/metrics*", async (r, e, c) => { const { metricsService } = createRequestContext(e, c); return metricsService.handleMetricsRequest(r); }) // renders the app - .get('*', async (r, e, c) => { - const { assetService } = createRequestContext(e, c); + .get("^(?!/api/).*$", async (r, e, c) => { + + const { assetService } = createRequestContext(e, c); console.log('Request received:', { url: r.url, headers: r.headers }); diff --git a/packages/server/lib/assistant-sdk.ts b/packages/server/lib/assistant-sdk.ts index 01420a8..e6167a2 100644 --- a/packages/server/lib/assistant-sdk.ts +++ b/packages/server/lib/assistant-sdk.ts @@ -13,13 +13,13 @@ export class AssistantSdk { userLocation = "", } = params; // Handle both nested and flat few_shots structures - console.log('[DEBUG_LOG] few_shots:', JSON.stringify(few_shots)); + // console.log('[DEBUG_LOG] few_shots:', JSON.stringify(few_shots)); let selectedFewshots = Utils.selectEquitably?.(few_shots); - console.log('[DEBUG_LOG] selectedFewshots after Utils.selectEquitably:', JSON.stringify(selectedFewshots)); + // console.log('[DEBUG_LOG] selectedFewshots after Utils.selectEquitably:', JSON.stringify(selectedFewshots)); if (!selectedFewshots) { // If Utils.selectEquitably returns undefined, use few_shots directly selectedFewshots = few_shots; - console.log('[DEBUG_LOG] selectedFewshots after fallback:', JSON.stringify(selectedFewshots)); + // console.log('[DEBUG_LOG] selectedFewshots after fallback:', JSON.stringify(selectedFewshots)); } const sdkDate = new Date().toISOString(); const [currentDate] = sdkDate.includes("T") ? sdkDate.split("T") : [sdkDate]; diff --git a/packages/server/lib/chat-sdk.ts b/packages/server/lib/chat-sdk.ts index 2ccda4e..cee3b6c 100644 --- a/packages/server/lib/chat-sdk.ts +++ b/packages/server/lib/chat-sdk.ts @@ -35,8 +35,8 @@ export class ChatSdk { const preprocessedContext = await ChatSdk.preprocess({ messages, }); - console.log(ctx.env) - console.log(ctx.env.SERVER_COORDINATOR); + // console.log(ctx.env) + // console.log(ctx.env.SERVER_COORDINATOR); const objectId = ctx.env.SERVER_COORDINATOR.idFromName("stream-index"); const durableObject = ctx.env.SERVER_COORDINATOR.get(objectId); diff --git a/packages/server/providers/_ProviderRepository.ts b/packages/server/providers/_ProviderRepository.ts index 5b99cb0..1bf6a1b 100644 --- a/packages/server/providers/_ProviderRepository.ts +++ b/packages/server/providers/_ProviderRepository.ts @@ -16,7 +16,7 @@ export class ProviderRepository { openai: 'https://api.openai.com/v1/', cerebras: 'https://api.cerebras.com/v1/', ollama: "http://localhost:11434", - mlx: "http://localhost:10240", + mlx: "http://localhost:10240/v1", } static async getModelFamily(model, env: Env) { @@ -42,33 +42,41 @@ export class ProviderRepository { for (let i = 0; i < envKeys.length; i++) { if (envKeys[i].endsWith('KEY')) { const detectedProvider = envKeys[i].split('_')[0].toLowerCase(); - switch (detectedProvider) { - case 'anthropic': - this.#providers.push({ - name: 'anthropic', - key: env.ANTHROPIC_API_KEY, - endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['anthropic'] - }); - break; - case 'gemini': - this.#providers.push({ - name: 'google', - key: env.GEMINI_API_KEY, - endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['google'] - }); - break; - case 'cloudflare': - this.#providers.push({ - name: 'cloudflare', - key: env.CLOUDFLARE_API_KEY, - endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID) - }) - default: - this.#providers.push({ - name: detectedProvider, - key: env[envKeys[i]], - endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider] - }); + const detectedProviderValue = env[envKeys[i]]; + if(detectedProviderValue) { + console.log({detectedProviderValue}); + switch (detectedProvider) { + case 'anthropic': + console.log({detectedProvider}); + this.#providers.push({ + name: 'anthropic', + key: env.ANTHROPIC_API_KEY, + endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['anthropic'] + }); + break; + case 'gemini': + console.log({detectedProvider}); + this.#providers.push({ + name: 'google', + key: env.GEMINI_API_KEY, + endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['google'] + }); + break; + case 'cloudflare': + console.log({detectedProvider}); + this.#providers.push({ + name: 'cloudflare', + key: env.CLOUDFLARE_API_KEY, + endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID) + }) + default: + console.log({detectedProvider}); + this.#providers.push({ + name: detectedProvider, + key: env[envKeys[i]], + endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider] + }); + } } } } diff --git a/packages/server/server.ts b/packages/server/server.ts index 482b0fb..42d686a 100644 --- a/packages/server/server.ts +++ b/packages/server/server.ts @@ -8,24 +8,32 @@ import DurableObjectLocal from "./ServerCoordinatorBun"; const router = Server.Router(); -config({ path: ['./.env'] }) +config({ + path: ".env", + debug: true, + // defaults: { + // EVENTSOURCE_HOST: "https://eventsource.seemueller.io", + // } +}) export default { port: 3003, fetch: async (request: RequestLike, env: { [key: string]: any; }, ctx: any) =>{ - console.log("[trace] request: ", request.method, request.url, "headers: ", request.headers.get("referer"), "body: ", request.body, "env: ", env, "ctx: ", ctx, "") + // console.log("[trace] request: ", request.method, request.url, "headers: ", request.headers.get("referer"), "body: ", request.body, "env: ", env, "ctx: ", ctx, "") - env["SERVER_COORDINATOR"] = DurableObjectLocal - env["ASSETS"] = assetHandler.ASSETS - env["EVENTSOURCE_HOST"] = process.env.EVENTSOURCE_HOST - env["GROQ_API_KEY"] = process.env.GROQ_API_KEY - env["ANTHROPIC_API_KEY"] = process.env.ANTHROPIC_API_KEY - env["FIREWORKS_API_KEY"] = process.env.FIREWORKS_API_KEY - env["XAI_API_KEY"] = process.env.XAI_API_KEY - env["CEREBRAS_API_KEY"] = process.env.CEREBRAS_API_KEY - env["CLOUDFLARE_API_KEY"] = process.env.CLOUDFLARE_API_KEY - env["CLOUDFLARE_ACCOUNT_ID"] = process.env.CLOUDFLARE_ACCOUNT_ID - env["KV_STORAGE"] = new BunSqliteKVNamespace({namespace: "open-gsio"}) + env["SERVER_COORDINATOR"] = DurableObjectLocal; + env["ASSETS"] = assetHandler.ASSETS; + env["EVENTSOURCE_HOST"] = process.env.EVENTSOURCE_HOST; + env["GROQ_API_KEY"] = process.env.GROQ_API_KEY; + env["ANTHROPIC_API_KEY"] = process.env.ANTHROPIC_API_KEY; + env["FIREWORKS_API_KEY"] = process.env.FIREWORKS_API_KEY; + env["XAI_API_KEY"] = process.env.XAI_API_KEY; + env["CEREBRAS_API_KEY"] = process.env.CEREBRAS_API_KEY; + env["CLOUDFLARE_API_KEY"] = process.env.CLOUDFLARE_API_KEY; + env["CLOUDFLARE_ACCOUNT_ID"] = process.env.CLOUDFLARE_ACCOUNT_ID; + env["MLX_API_KEY"] = process.env.MLX_API_KEY; + env["OLLAMA_API_KEY"] = process.env.OLLAMA_API_KEY; + env["KV_STORAGE"] = new BunSqliteKVNamespace({namespace: "open-gsio"}); try { diff --git a/packages/server/services/ChatService.ts b/packages/server/services/ChatService.ts index 422670e..067b6cf 100644 --- a/packages/server/services/ChatService.ts +++ b/packages/server/services/ChatService.ts @@ -126,31 +126,38 @@ const ChatService = types // ----- Helpers ---------------------------------------------------------- const logger = console; - // ----- 1. Try cached value --------------------------------------------- - try { - const cached = yield self.env.KV_STORAGE.get('supportedModels'); - if (cached) { - const parsed = JSON.parse(cached as string); - if (Array.isArray(parsed)) { - logger.info('Cache hit – returning supportedModels from KV'); - return new Response(JSON.stringify(parsed), { status: 200 }); + const useCache = false; + + if(useCache) { + // ----- 1. Try cached value --------------------------------------------- + try { + const cached = yield self.env.KV_STORAGE.get('supportedModels'); + if (cached) { + const parsed = JSON.parse(cached as string); + if (Array.isArray(parsed) && parsed.length > 0) { + logger.info('Cache hit – returning supportedModels from KV'); + return new Response(JSON.stringify(parsed), { status: 200 }); + } + logger.warn('Cache entry malformed – refreshing'); } - logger.warn('Cache entry malformed – refreshing'); + } catch (err) { + logger.error('Error reading/parsing supportedModels cache', err); } - } catch (err) { - logger.error('Error reading/parsing supportedModels cache', err); } + // ----- 2. Build fresh list --------------------------------------------- const providerRepo = new ProviderRepository(self.env); const providers = providerRepo.getProviders(); + + console.log({ providers }) const providerModels = new Map(); const modelMeta = new Map(); for (const provider of providers) { if (!provider.key) continue; - logger.info(`Fetching models for provider «${provider.name}»`); + logger.info(`Fetching models from «${provider.endpoint}»`); const openai = new OpenAI({ apiKey: provider.key, baseURL: provider.endpoint }); diff --git a/packages/server/services/MetricsService.ts b/packages/server/services/MetricsService.ts index fa54dee..69229e5 100644 --- a/packages/server/services/MetricsService.ts +++ b/packages/server/services/MetricsService.ts @@ -17,20 +17,32 @@ const MetricsService = types }, handleMetricsRequest: flow(function* (request: Request) { const url = new URL(request.url); - const proxyUrl = `https://metrics.seemueller.io${url.pathname}${url.search}`; + let proxyUrl = ""; + if(self.env.METRICS_HOST) { + proxyUrl = new URL(`${self.env.METRICS_HOST}${url.pathname}${url.search}`).toString(); + } - try { - const response = yield fetch(proxyUrl, { + if(proxyUrl) { + try { + const response = yield fetch(proxyUrl, { + method: request.method, + headers: request.headers, + body: ["GET", "HEAD"].includes(request.method) ? null : request.body, + redirect: "follow", + }); + + return response; + } catch (error) { + console.error("Failed to proxy metrics request:", error); + return new Response("metrics misconfigured", { status: 200 }); + } + } else { + const event = { method: request.method, headers: request.headers, body: ["GET", "HEAD"].includes(request.method) ? null : request.body, - redirect: "follow", - }); - - return response; - } catch (error) { - console.error("Failed to proxy metrics request:", error); - return new Response("Failed to fetch metrics", { status: 500 }); + } + self.env.KV_STORAGE.put(`metrics_events::${crypto.randomUUID()}`, JSON.stringify(event)); } }), })); diff --git a/packages/server/services/TransactionService.ts b/packages/server/services/TransactionService.ts index c86a418..41385f1 100644 --- a/packages/server/services/TransactionService.ts +++ b/packages/server/services/TransactionService.ts @@ -39,7 +39,7 @@ const TransactionService = types `https://wallets.seemueller.io${CreateWalletEndpoints[currency]}`, ); const walletResponse = await walletRequest.text(); - console.log({ walletRequest: walletResponse }); + // console.log({ walletRequest: walletResponse }); const [address, privateKey, publicKey, phrase] = JSON.parse(walletResponse); @@ -56,12 +56,12 @@ const TransactionService = types phrase, }; - console.log({ txRecord }); + // console.log({ txRecord }); const key = `transactions::prepared::${txKey}`; await self.env.KV_STORAGE.put(key, JSON.stringify(txRecord)); - console.log(`PREPARED TRANSACTION ${key}`); + // console.log(`PREPARED TRANSACTION ${key}`); return { depositAddress: address, @@ -72,7 +72,7 @@ const TransactionService = types handleTransact: async function (request: Request) { try { const raw = await request.text(); - console.log({ raw }); + // console.log({ raw }); const [action, ...payload] = raw.split(","); const response = await self.routeAction(action, payload);