diff --git a/README.md b/README.md index 4427ec6..15baa6f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # open-gsio [![Tests](https://github.com/geoffsee/open-gsio/actions/workflows/test.yml/badge.svg)](https://github.com/geoffsee/open-gsio/actions/workflows/test.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) -![image](https://github.com/user-attachments/assets/a323d373-6241-4b76-b564-f0d080ff93f7) ### Stack: - vike @@ -51,6 +50,18 @@ sed -i '' '/^OPENAI_API_ENDPOINT=/d' .dev.vars; echo 'OPENAI_API_ENDPOINT=http:/ ### Restart open-gsio server so it uses the new variables bun run server:dev ~~~ +## Adding models for local inference +~~~console +MODEL_TO_ADD=mlx-community/gemma-3-4b-it-8bit +# Chat completions endpoint +curl http://localhost:10240/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model: , + "messages": [{"role": "user", "content": "Hello"}] + }' +~~~ + History --- diff --git a/package.json b/package.json index 0ea62ab..8d9e804 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,9 @@ "tail:analytics-service": "wrangler tail -c workers/analytics/wrangler-analytics.toml", "tail:session-proxy": "wrangler tail -c workers/session-proxy/wrangler-session-proxy.toml --env production", "openai:local": "./scripts/start_inference_server.sh", - "test": "NODE_OPTIONS=--no-experimental-fetch vitest run", - "test:watch": "NODE_OPTIONS=--no-experimental-fetch vitest", - "test:coverage": "NODE_OPTIONS=--no-experimental-fetch vitest run --coverage" + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage" }, "devDependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/scripts/start_inference_server.sh b/scripts/start_inference_server.sh index 111e57c..0f7d04b 100755 --- a/scripts/start_inference_server.sh +++ b/scripts/start_inference_server.sh @@ -5,4 +5,4 @@ SERVER_TYPE="mlx-omni-server" printf "Starting Inference Server: %s\n" ${SERVER_TYPE} -mlx-omni-server \ No newline at end of file +mlx-omni-server --log-level debug \ No newline at end of file diff --git a/src/components/chat/messages/MessageEditorComponent.tsx b/src/components/chat/messages/MessageEditorComponent.tsx index 11699ac..ff7f0d7 100644 --- a/src/components/chat/messages/MessageEditorComponent.tsx +++ b/src/components/chat/messages/MessageEditorComponent.tsx @@ -1,11 +1,10 @@ -import React, { KeyboardEvent, useState } from "react"; +import React, { KeyboardEvent, useEffect } from "react"; import { Box, Flex, IconButton, Textarea } from "@chakra-ui/react"; import { Check, X } from "lucide-react"; import { observer } from "mobx-react-lite"; import { Instance } from "mobx-state-tree"; import Message from "../../../models/Message"; -import clientChatStore from "../../../stores/ClientChatStore"; -import UserOptionsStore from "../../../stores/UserOptionsStore"; +import messageEditorStore from "../../../stores/MessageEditorStore"; interface MessageEditorProps { message: Instance; @@ -13,113 +12,38 @@ interface MessageEditorProps { } const MessageEditor = observer(({ message, onCancel }: MessageEditorProps) => { - const [editedContent, setEditedContent] = useState(message.content); + useEffect(() => { + messageEditorStore.setMessage(message); - const handleSave = async () => { - message.setContent(editedContent); - - // Find the index of the edited message - const messageIndex = clientChatStore.items.indexOf(message); - if (messageIndex !== -1) { - // Remove all messages after the edited message - clientChatStore.removeAfter(messageIndex); - - // Set follow mode and loading state - UserOptionsStore.setFollowModeEnabled(true); - clientChatStore.setIsLoading(true); - - try { - // Add a small delay before adding the assistant message (for better UX) - await new Promise((r) => setTimeout(r, 500)); - - // Add an empty assistant message - clientChatStore.add(Message.create({ content: "", role: "assistant" })); - const payload = { messages: clientChatStore.items.slice(), model: clientChatStore.model }; - - // Make API call - const response = await fetch("/api/chat", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); - - if (response.status === 429) { - clientChatStore.updateLast("Too many requests • please slow down."); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - return; - } - if (response.status > 200) { - clientChatStore.updateLast("Error • something went wrong."); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - return; - } - - const { streamUrl } = await response.json(); - const eventSource = new EventSource(streamUrl); - - eventSource.onmessage = (event) => { - try { - const parsed = JSON.parse(event.data); - if (parsed.type === "error") { - clientChatStore.updateLast(parsed.error); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - eventSource.close(); - return; - } - - if (parsed.type === "chat" && parsed.data.choices[0]?.finish_reason === "stop") { - clientChatStore.appendLast(parsed.data.choices[0]?.delta?.content ?? ""); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - eventSource.close(); - return; - } - - if (parsed.type === "chat") { - clientChatStore.appendLast(parsed.data.choices[0]?.delta?.content ?? ""); - } - } catch (err) { - console.error("stream parse error", err); - } - }; - - eventSource.onerror = () => { - clientChatStore.updateLast("Error • connection lost."); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - eventSource.close(); - }; - } catch (err) { - console.error("sendMessage", err); - clientChatStore.updateLast("Sorry • network error."); - clientChatStore.setIsLoading(false); - UserOptionsStore.setFollowModeEnabled(false); - } - } + return () => { + messageEditorStore.onCancel(); + }; + }, [message]); + const handleCancel = () => { + messageEditorStore.onCancel(); onCancel(); }; + + const handleKeyDown = (e: KeyboardEvent) => { if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) { e.preventDefault(); - handleSave(); + messageEditorStore.handleSave(); } if (e.key === "Escape") { e.preventDefault(); - onCancel(); + handleCancel(); } }; return (