Remove unused services and refactor SDK structure

Deleted outdated SDKs and services, including DocumentService and markdown-sdk. Consolidated and relocated SDKs into a unified "providers" structure to improve maintainability. Updated imports and adjusted utils naming for consistency.

workers/site/context.ts

@@ -4,7 +4,6 @@ import AssetService from "./services/AssetService";
 import MetricsService from "./services/MetricsService";
 import ChatService from "./services/ChatService";
 import TransactionService from "./services/TransactionService";
-import DocumentService from "./services/DocumentService";
 import FeedbackService from "./services/FeedbackService";
 
 const Context = types
@@ -14,7 +13,6 @@ const Context = types
     assetService: types.optional(AssetService, {}),
     metricsService: types.optional(MetricsService, {}),
     transactionService: types.optional(TransactionService, {}),
-    documentService: types.optional(DocumentService, {}),
     feedbackService: types.optional(FeedbackService, {}),
   })
   .actions((self) => {
@@ -45,7 +43,6 @@ const createServerContext = (env, ctx) => {
     contactService: ContactService.create({}),
     assetService: AssetService.create({}),
     transactionService: TransactionService.create({}),
-    documentService: DocumentService.create({}),
     feedbackService: FeedbackService.create({}),
     metricsService: MetricsService.create({
       isCollectingMetrics: true,

workers/site/providers/cerebras.ts

@@ -6,7 +6,7 @@ import {
   ModelSnapshotType2,
   UnionStringArray,
 } from "mobx-state-tree";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class CerebrasSdk {
   static async handleCerebrasStream(

workers/site/providers/claude.ts

@@ -7,7 +7,7 @@ import {
   ModelSnapshotType2,
   UnionStringArray,
 } from "mobx-state-tree";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class ClaudeChatSdk {
   private static async streamClaudeResponse(

workers/site/providers/cloudflareAi.ts

@@ -6,7 +6,7 @@ import {
   ModelSnapshotType2,
   UnionStringArray,
 } from "mobx-state-tree";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class CloudflareAISdk {
   static async handleCloudflareAIStream(

workers/site/providers/fireworks.ts

@@ -8,9 +8,8 @@ import {
   ModelSnapshotType2,
   UnionStringArray,
 } from "mobx-state-tree";
-import Message from "../../models/Message";
-import { MarkdownSdk } from "../markdown-sdk";
-import ChatSdk from "../chat-sdk";
+import Message from "../models/Message";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class FireworksAiChatSdk {
   private static async streamFireworksResponse(

workers/site/providers/google.ts

@@ -1,6 +1,6 @@
 import { OpenAI } from "openai";
-import ChatSdk from "../chat-sdk";
-import { StreamParams } from "../../services/ChatService";
+import ChatSdk from "../sdk/chat-sdk";
+import { StreamParams } from "../services/ChatService";
 
 export class GoogleChatSdk {
   static async handleGoogleStream(

workers/site/providers/groq.ts

@@ -6,7 +6,7 @@ import {
   ModelSnapshotType2,
   UnionStringArray,
 } from "mobx-state-tree";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class GroqChatSdk {
   static async handleGroqStream(

workers/site/providers/openai.ts

@@ -1,5 +1,5 @@
 import { OpenAI } from "openai";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class OpenAiChatSdk {
   static async handleOpenAiStream(

workers/site/providers/xai.ts

@@ -1,5 +1,5 @@
 import { OpenAI } from "openai";
-import ChatSdk from "../chat-sdk";
+import ChatSdk from "../sdk/chat-sdk";
 
 export class XaiChatSdk {
   static async handleXaiStream(

workers/site/sdk/assistant-sdk.ts

@@ -1,4 +1,4 @@
-import { Sdk } from "./sdk";
+import { Utils } from "./utils";
 import few_shots from "../prompts/few_shots";
 
 export class AssistantSdk {
@@ -12,10 +12,10 @@ export class AssistantSdk {
       userTimezone = "UTC",
       userLocation = "",
     } = params;
-    const selectedFewshots = Sdk.selectEquitably?.(few_shots) || few_shots;
+    const selectedFewshots = Utils.selectEquitably?.(few_shots) || few_shots;
     const sdkDate =
-      typeof Sdk.getCurrentDate === "function"
-        ? Sdk.getCurrentDate()
+      typeof Utils.getCurrentDate === "function"
+        ? Utils.getCurrentDate()
         : new Date().toISOString();
     const [currentDate] = sdkDate.split("T");
     const now = new Date();

workers/site/sdk/markdown-sdk.ts

@@ -1,54 +0,0 @@
-export class MarkdownSdk {
-  static formatContextContainer(contextContainer) {
-    let markdown = "# Assistant Tools Results\n\n";
-
-    for (const [key, value] of contextContainer.entries()) {
-      markdown += `## ${this._escapeForMarkdown(key)}\n\n`;
-      markdown += this._formatValue(value);
-    }
-
-    return markdown.trim();
-  }
-
-  static _formatValue(value, depth = 0) {
-    if (Array.isArray(value)) {
-      return this._formatArray(value, depth);
-    } else if (value && typeof value === "object") {
-      return this._formatObject(value, depth);
-    } else {
-      return this._formatPrimitive(value, depth);
-    }
-  }
-
-  static _formatArray(arr, depth) {
-    let output = "";
-    arr.forEach((item, i) => {
-      output += `### Item ${i + 1}\n`;
-      output += this._formatValue(item, depth + 1);
-      output += "\n";
-    });
-    return output;
-  }
-
-  static _formatObject(obj, depth) {
-    return (
-      Object.entries(obj)
-        .map(
-          ([k, v]) =>
-            `- **${this._escapeForMarkdown(k)}**: ${this._escapeForMarkdown(v)}`,
-        )
-        .join("\n") + "\n\n"
-    );
-  }
-
-  static _formatPrimitive(value, depth) {
-    return `${this._escapeForMarkdown(String(value))}\n\n`;
-  }
-
-  static _escapeForMarkdown(text) {
-    if (typeof text !== "string") {
-      text = String(text);
-    }
-    return text.replace(/(\*|`|_|~)/g, "\\$1");
-  }
-}

workers/site/sdk/message-sdk.ts

@@ -1,156 +0,0 @@
-interface BaseMessage {
-  role: "user" | "assistant" | "system";
-}
-
-interface TextMessage extends BaseMessage {
-  content: string;
-}
-
-interface O1Message extends BaseMessage {
-  content: Array<{
-    type: string;
-    text: string;
-  }>;
-}
-
-interface LlamaMessage extends BaseMessage {
-  content: Array<{
-    type: "text" | "image";
-    data: string;
-  }>;
-}
-
-interface MessageConverter<T extends BaseMessage, U extends BaseMessage> {
-  convert(message: T): U;
-  convertBatch(messages: T[]): U[];
-}
-
-class TextToO1Converter implements MessageConverter<TextMessage, O1Message> {
-  convert(message: TextMessage): O1Message {
-    return {
-      role: message.role,
-      content: [
-        {
-          type: "text",
-          text: message.content,
-        },
-      ],
-    };
-  }
-
-  convertBatch(messages: TextMessage[]): O1Message[] {
-    return messages.map((msg) => this.convert(msg));
-  }
-}
-
-class O1ToTextConverter implements MessageConverter<O1Message, TextMessage> {
-  convert(message: O1Message): TextMessage {
-    return {
-      role: message.role,
-      content: message.content.map((item) => item.text).join("\n"),
-    };
-  }
-
-  convertBatch(messages: O1Message[]): TextMessage[] {
-    return messages.map((msg) => this.convert(msg));
-  }
-}
-
-class TextToLlamaConverter
-  implements MessageConverter<TextMessage, LlamaMessage>
-{
-  convert(message: TextMessage): LlamaMessage {
-    return {
-      role: message.role,
-      content: [
-        {
-          type: "text",
-          data: message.content,
-        },
-      ],
-    };
-  }
-
-  convertBatch(messages: TextMessage[]): LlamaMessage[] {
-    return messages.map((msg) => this.convert(msg));
-  }
-}
-
-class LlamaToTextConverter
-  implements MessageConverter<LlamaMessage, TextMessage>
-{
-  convert(message: LlamaMessage): TextMessage {
-    return {
-      role: message.role,
-      content: message.content
-        .filter((item) => item.type === "text")
-        .map((item) => item.data)
-        .join("\n"),
-    };
-  }
-
-  convertBatch(messages: LlamaMessage[]): TextMessage[] {
-    return messages.map((msg) => this.convert(msg));
-  }
-}
-
-class MessageConverterFactory {
-  static createConverter(
-    fromFormat: string,
-    toFormat: string,
-  ): MessageConverter<any, any> {
-    const key = `${fromFormat}->${toFormat}`;
-    const converters = {
-      "text->o1": new TextToO1Converter(),
-      "o1->text": new O1ToTextConverter(),
-      "text->llama": new TextToLlamaConverter(),
-      "llama->text": new LlamaToTextConverter(),
-    };
-
-    const converter = converters[key];
-    if (!converter) {
-      throw new Error(`Unsupported conversion: ${key}`);
-    }
-
-    return converter;
-  }
-}
-
-function detectMessageFormat(message: any): string {
-  if (typeof message.content === "string") {
-    return "text";
-  }
-  if (Array.isArray(message.content)) {
-    if (message.content[0]?.type === "text" && "text" in message.content[0]) {
-      return "o1";
-    }
-    if (message.content[0]?.type && "data" in message.content[0]) {
-      return "llama";
-    }
-  }
-  throw new Error("Unknown message format");
-}
-
-function convertMessage(message: any, targetFormat: string): any {
-  const sourceFormat = detectMessageFormat(message);
-  if (sourceFormat === targetFormat) {
-    return message;
-  }
-
-  const converter = MessageConverterFactory.createConverter(
-    sourceFormat,
-    targetFormat,
-  );
-  return converter.convert(message);
-}
-
-export {
-  MessageConverterFactory,
-  convertMessage,
-  detectMessageFormat,
-  type BaseMessage,
-  type TextMessage,
-  type O1Message,
-  type LlamaMessage,
-  type MessageConverter,
-};

workers/site/sdk/perigon-sdk.ts

@@ -1,97 +0,0 @@
-export interface AdvancedSearchParams {
-  mainQuery?: string;
-  titleQuery?: string;
-  descriptionQuery?: string;
-  contentQuery?: string;
-  mustInclude?: string[];
-  mustNotInclude?: string[];
-  exactPhrases?: string[];
-  urlContains?: string;
-}
-
-export class PerigonSearchBuilder {
-  private buildExactPhraseQuery(phrases: string[]): string {
-    return phrases.map((phrase) => `"${phrase}"`).join(" AND ");
-  }
-
-  private buildMustIncludeQuery(terms: string[]): string {
-    return terms.join(" AND ");
-  }
-
-  private buildMustNotIncludeQuery(terms: string[]): string {
-    return terms.map((term) => `NOT ${term}`).join(" AND ");
-  }
-
-  buildSearchParams(params: AdvancedSearchParams): SearchParams {
-    const searchParts: string[] = [];
-    const searchParams: SearchParams = {};
-
-    if (params.mainQuery) {
-      searchParams.q = params.mainQuery;
-    }
-
-    if (params.titleQuery) {
-      searchParams.title = params.titleQuery;
-    }
-
-    if (params.descriptionQuery) {
-      searchParams.desc = params.descriptionQuery;
-    }
-
-    if (params.contentQuery) {
-      searchParams.content = params.contentQuery;
-    }
-
-    if (params.exactPhrases?.length) {
-      searchParts.push(this.buildExactPhraseQuery(params.exactPhrases));
-    }
-
-    if (params.mustInclude?.length) {
-      searchParts.push(this.buildMustIncludeQuery(params.mustInclude));
-    }
-
-    if (params.mustNotInclude?.length) {
-      searchParts.push(this.buildMustNotIncludeQuery(params.mustNotInclude));
-    }
-
-    if (searchParts.length) {
-      searchParams.q = searchParams.q
-        ? `(${searchParams.q}) AND (${searchParts.join(" AND ")})`
-        : searchParts.join(" AND ");
-    }
-
-    if (params.urlContains) {
-      searchParams.url = `"${params.urlContains}"`;
-    }
-
-    return searchParams;
-  }
-}
-
-export interface SearchParams {
-  /** Main search query parameter that searches across title, description and content */
-  q?: string;
-  /** Search only in article titles */
-  title?: string;
-  /** Search only in article descriptions */
-  desc?: string;
-  /** Search only in article content */
-  content?: string;
-  /** Search in article URLs */
-  url?: string;
-  /** Additional search parameters can be added here as needed */
-  [key: string]: string | undefined;
-}
-
-export interface Article {
-  translation: {
-    title: string;
-    description: string;
-    content: string;
-    url: string;
-  };
-}
-
-export interface SearchResponse {
-  articles?: Article[];
-}

workers/site/sdk/stream-processor-sdk.ts

@@ -1,38 +0,0 @@
-export class StreamProcessorSdk {
-  static preprocessContent(buffer: string): string {
-    return buffer
-      .replace(/(\n\- .*\n)+/g, "$&\n")
-      .replace(/(\n\d+\. .*\n)+/g, "$&\n")
-      .replace(/\n{3,}/g, "\n\n");
-  }
-
-  static async handleStreamProcessing(
-    stream: any,
-    controller: ReadableStreamDefaultController,
-  ) {
-    const encoder = new TextEncoder();
-    let buffer = "";
-
-    try {
-      for await (const chunk of stream) {
-        const content = chunk.choices[0]?.delta?.content || "";
-        buffer += content;
-
-        let processedContent = StreamProcessorSdk.preprocessContent(buffer);
-        controller.enqueue(encoder.encode(processedContent));
-
-        buffer = "";
-      }
-
-      if (buffer) {
-        let processedContent = StreamProcessorSdk.preprocessContent(buffer);
-        controller.enqueue(encoder.encode(processedContent));
-      }
-    } catch (error) {
-      controller.error(error);
-      throw new Error("Stream processing error");
-    } finally {
-      controller.close();
-    }
-  }
-}

workers/site/sdk/utils.ts

@@ -1,4 +1,4 @@
-export class Sdk {
+export class Utils {
   static getSeason(date: string): string {
     const hemispheres = {
       Northern: ["Winter", "Spring", "Summer", "Autumn"],

workers/site/services/ChatService.ts

@@ -4,15 +4,15 @@ import ChatSdk from '../sdk/chat-sdk';
 import Message from "../models/Message";
 import O1Message from "../models/O1Message";
 import {getModelFamily, ModelFamily} from "../../../src/components/chat/lib/SupportedModels";
-import {OpenAiChatSdk} from "../sdk/models/openai";
-import {GroqChatSdk} from "../sdk/models/groq";
-import {ClaudeChatSdk} from "../sdk/models/claude";
-import {FireworksAiChatSdk} from "../sdk/models/fireworks";
+import {OpenAiChatSdk} from "../providers/openai";
+import {GroqChatSdk} from "../providers/groq";
+import {ClaudeChatSdk} from "../providers/claude";
+import {FireworksAiChatSdk} from "../providers/fireworks";
 import handleStreamData from "../sdk/handleStreamData";
-import {GoogleChatSdk} from "../sdk/models/google";
-import {XaiChatSdk} from "../sdk/models/xai";
-import {CerebrasSdk} from "../sdk/models/cerebras";
-import {CloudflareAISdk} from "../sdk/models/cloudflareAi";
+import {GoogleChatSdk} from "../providers/google";
+import {XaiChatSdk} from "../providers/xai";
+import {CerebrasSdk} from "../providers/cerebras";
+import {CloudflareAISdk} from "../providers/cloudflareAi";
 
 export interface StreamParams {
     env: Env;

workers/site/services/DocumentService.ts

@@ -1,145 +0,0 @@
-import { flow, types } from "mobx-state-tree";
-
-async function getExtractedText(file: any) {
-  const formData = new FormData();
-  formData.append("file", file);
-
-  const response = await fetch("https://any2text.seemueller.io/extract", {
-    method: "POST",
-    body: formData,
-  });
-
-  if (!response.ok) {
-    throw new Error(`Failed to extract text: ${response.statusText}`);
-  }
-
-  const { text: extractedText } = await response.json<{ text: string }>();
-  return extractedText;
-}
-
-export default types
-  .model("DocumentService", {})
-  .volatile(() => ({
-    env: {} as Env,
-    ctx: {} as ExecutionContext,
-  }))
-  .actions((self) => ({
-    setEnv(env: Env) {
-      self.env = env;
-    },
-    setCtx(ctx: ExecutionContext) {
-      self.ctx = ctx;
-    },
-    handlePutDocument: flow(function* (request: Request) {
-      try {
-        if (!request.body) {
-          return new Response("No content in the request", { status: 400 });
-        }
-
-        const formData = yield request.formData();
-        const file = formData.get("file");
-        const name = file instanceof File ? file.name : "unnamed";
-
-        if (!(file instanceof File)) {
-          return new Response("No valid file found in form data", {
-            status: 400,
-          });
-        }
-
-        const key = `document_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-
-        const content = yield file.arrayBuffer();
-
-        const contentType = file.type || "application/octet-stream";
-        const contentLength = content.byteLength;
-
-        const metadata = {
-          name,
-          contentType,
-          contentLength,
-          uploadedAt: new Date().toISOString(),
-        };
-
-        yield self.env.KV_STORAGE.put(key, content, {
-          expirationTtl: 60 * 60 * 24 * 7,
-        });
-
-        yield self.env.KV_STORAGE.put(`${key}_meta`, JSON.stringify(metadata), {
-          expirationTtl: 60 * 60 * 24 * 7,
-        });
-
-        const url = new URL(request.url);
-        url.pathname = `/api/documents/${key}`;
-
-        console.log(content.length);
-        const extracted = yield getExtractedText(file);
-
-        console.log({ extracted });
-
-        return new Response(
-          JSON.stringify({
-            url: url.toString(),
-            name,
-            extractedText: extracted,
-          }),
-          { status: 200 },
-        );
-      } catch (error) {
-        console.error("Error uploading document:", error);
-        return new Response("Failed to upload document", { status: 500 });
-      }
-    }),
-    handleGetDocument: flow(function* (request: Request) {
-      try {
-        const url = new URL(request.url);
-        const key = url.pathname.split("/").pop();
-
-        if (!key) {
-          return new Response("Document key is missing", { status: 400 });
-        }
-
-        const content = yield self.env.KV_STORAGE.get(key, "arrayBuffer");
-
-        if (!content) {
-          return new Response("Document not found", { status: 404 });
-        }
-
-        return new Response(content, {
-          status: 200,
-          headers: {
-            "Content-Type": "application/octet-stream",
-            "Content-Disposition": `attachment; filename="${key}"`,
-          },
-        });
-      } catch (error) {
-        console.error("Error retrieving document:", error);
-        return new Response("Failed to retrieve document", { status: 500 });
-      }
-    }),
-    handleGetDocumentMeta: flow(function* (request: Request) {
-      try {
-        const url = new URL(request.url);
-        const key = url.pathname.split("/").pop();
-
-        if (!key) {
-          return new Response("Document key is missing", { status: 400 });
-        }
-
-        const content = yield self.env.KV_STORAGE.get(`${key}_meta`);
-
-        if (!content) {
-          return new Response("Document meta not found", { status: 404 });
-        }
-
-        return new Response(JSON.stringify({ metadata: content }), {
-          status: 200,
-          headers: {
-            "Content-Type": "application/json",
-          },
-        });
-      } catch (error) {
-        console.error("Error retrieving document:", error);
-        return new Response("Failed to retrieve document", { status: 500 });
-      }
-    }),
-  }));

workers/site/worker.ts

@@ -1,8 +1,8 @@
 import { createRouter } from "./api-router";
 import SiteCoordinator from "./durable_objects/SiteCoordinator";
 
-// exports durable object
+// durable object
 export { SiteCoordinator };
 
-// exports worker
+// worker
 export default createRouter();