move to workspaces

packages/toak/.gitignore

@@ -0,0 +1,2 @@
+prompt.md
+.toak-ignore

packages/toak/.prettierignore

@@ -0,0 +1,3 @@
+# Ignore artifacts:
+build
+coverage

packages/toak/.prettierrc

@@ -0,0 +1,32 @@
+{
+  "semi": true,
+  "singleQuote": true,
+  "trailingComma": "es5",
+  "tabWidth": 2,
+  "printWidth": 100,
+  "bracketSpacing": true,
+  "arrowParens": "avoid",
+  "endOfLine": "lf",
+  "jsxSingleQuote": false,
+  "quoteProps": "as-needed",
+  "bracketSameLine": false,
+  "proseWrap": "preserve",
+  "htmlWhitespaceSensitivity": "css",
+  "embeddedLanguageFormatting": "auto",
+  "singleAttributePerLine": false,
+  "overrides": [
+    {
+      "files": "*.md",
+      "options": {
+        "proseWrap": "always",
+        "printWidth": 80
+      }
+    },
+    {
+      "files": ["*.yml", "*.yaml"],
+      "options": {
+        "singleQuote": false
+      }
+    }
+  ]
+}

packages/toak/build.ts

@@ -0,0 +1,22 @@
+import isolatedDecl from 'bun-plugin-isolated-decl';
+
+// handles building the library
+await Bun.build({
+  entrypoints: [
+    "src/cli.ts",
+    "src/fileExclusions.ts",
+    "src/fileTypeExclusions.ts",
+    "src/index.ts",
+    "src/MarkdownGenerator.ts",
+    "src/TokenCleaner.ts"
+  ],
+  outdir: './dist',
+  minify: true,
+  target: 'node',
+  splitting: true,
+  plugins: [
+    isolatedDecl({
+      forceGenerate: true,  // Generate declaration files even if there are errors
+    })
+  ],
+});

packages/toak/eslint.config.js

@@ -0,0 +1,74 @@
+import globals from 'globals';
+import js from '@eslint/js';
+
+/** @type {import('eslint').Linter.Config[]} */
+export default [
+  {
+    ignores: ['dist/**', 'node_modules/**', '*.min.js', '*.d.ts'],
+  },
+  {
+    languageOptions: {
+      ecmaVersion: 2024,
+      sourceType: 'module',
+      globals: {
+        ...globals.node,
+        ...globals.browser,
+      },
+    },
+    linterOptions: {
+      reportUnusedDisableDirectives: true,
+    },
+    rules: {
+      ...js.configs.recommended.rules,
+      'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off',
+      'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off',
+      'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+      'no-constant-condition': ['error', { checkLoops: false }],
+      'no-multiple-empty-lines': ['error', { max: 1, maxEOF: 0 }],
+      quotes: ['error', 'single', { avoidEscape: true }],
+      semi: ['error', 'always'],
+      indent: ['error', 2, { SwitchCase: 1 }],
+      'comma-dangle': ['error', 'always-multiline'],
+      'arrow-parens': ['error', 'as-needed'], // Changed from 'avoid' to 'as-needed'
+      'object-curly-spacing': ['error', 'always'],
+      'array-bracket-spacing': ['error', 'never'],
+      'space-before-function-paren': [
+        'error',
+        {
+          anonymous: 'never',
+          named: 'never',
+          asyncArrow: 'always',
+        },
+      ],
+      'no-trailing-spaces': 'error',
+      'eol-last': ['error', 'always'],
+      'prefer-const': 'error',
+      'no-var': 'error',
+      eqeqeq: ['error', 'always'],
+      curly: ['error', 'all'],
+      'brace-style': ['error', '1tbs', { allowSingleLine: false }],
+      'keyword-spacing': ['error', { before: true, after: true }],
+      'space-infix-ops': 'error',
+      'comma-spacing': ['error', { before: false, after: true }],
+      'no-multi-spaces': 'error',
+      'no-irregular-whitespace': 'error',
+      'no-mixed-spaces-and-tabs': 'error',
+      'no-else-return': 'error',
+    },
+  },
+  {
+    files: ['src/TokenCleaner.js'],
+    rules: {
+      'no-useless-escape': 'off',
+    },
+  },
+  {
+    files: ['**/*.test.js', '**/*.spec.js'],
+
+    languageOptions: {
+      globals: {
+        ...globals.jest,
+      },
+    },
+  },
+];

packages/toak/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "toak",
+  "version": "3.1.3",
+  "type": "module",
+  "license": "AGPL-3.0-or-later",
+  "repository": "https://github.com/seemueller-io/toak.git",
+  "author": "Geoff Seemueller",
+  "description": "Open-source CLI/Library for tokenizing a git repository into a markdown file.",
+  "keywords": [
+    "cli",
+    "tool",
+    "utility",
+    "tokenize",
+    "git",
+    "repository",
+    "code",
+    "export",
+    "markdown",
+    "prompt",
+    "ai",
+    "llm"
+  ],
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "bin": {
+    "toak": "./dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "rm -rf dist && bun ./build.ts",
+    "test": "bun test",
+    "prepublishOnly": "bun run build",
+    "dev": "bun run .",
+    "deploy:dev": "bun run build && bun publish .",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
+    "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md,yml,yaml}\"",
+    "fix": "bun format && bun lint:fix",
+    "release": "bunx release-it"
+  },
+  "dependencies": {
+    "glob": "^11.0.1",
+    "llama3-tokenizer-js": "^1.2.0",
+    "micromatch": "^4.0.8"
+  },
+  "peerDependencies": {
+    "node": "^20.18.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.24.0",
+    "@types/bun": "latest",
+    "@types/micromatch": "^4.0.9",
+    "@types/node": "^22.14.0",
+    "bun": "latest",
+    "bun-plugin-isolated-decl": "^0.1.10",
+    "eslint": "^9.24.0",
+    "globals": "^15.15.0",
+    "oxc-transform": "^0.44.0",
+    "prettier": "^3.5.3",
+    "typescript": "^5.8.3"
+  },
+  "release-it": {
+    "$schema": "https://unpkg.com/release-it/schema/release-it.json",
+    "git": {
+      "commitMessage": "Release v${version}"
+    },
+    "github": {
+      "release": true
+    },
+    "npm": {
+      "release": true
+    }
+  }
+}

packages/toak/src/MarkdownGenerator.ts

@@ -0,0 +1,363 @@
+import path from 'path';
+import { execSync } from 'child_process';
+import { readFile, writeFile } from 'fs/promises';
+import llama3Tokenizer from 'llama3-tokenizer-js';
+import { TokenCleaner } from './TokenCleaner.js';
+import * as micromatch from 'micromatch';
+import fileTypeExclusions from './fileTypeExclusions.js';
+import fileExclusions from './fileExclusions.js';
+import { readFileSync } from 'node:fs';
+import { glob } from 'glob';
+import { isPreset, type PresetPrompt, prompts } from './prompts.ts';
+
+
+export interface MarkdownGeneratorOptions {
+  dir?: string;
+  outputFilePath?: string;
+  fileTypeExclusions?: Set<string>;
+  fileExclusions?: string[];
+  customPatterns?: Record<string, any>;
+  customSecretPatterns?: Record<string, any>;
+  verbose?: boolean;
+  todoPrompt?: string
+}
+
+/**
+ * @class MarkdownGenerator
+ * @description A class that generates markdown documentation from tracked Git files in a project.
+ * It can exclude specific file types and files, clean tokens, and include todo lists.
+ */
+export class MarkdownGenerator {
+  private dir: string;
+  private outputFilePath: string;
+  private fileTypeExclusions: Set<string>;
+  private fileExclusions: string[];
+  private tokenCleaner: TokenCleaner;
+  private verbose: boolean;
+  private initialized: boolean;
+  private todoPrompt: string;
+
+  /**
+   * Creates an instance of MarkdownGenerator.
+   * @param {MarkdownGeneratorOptions} [options={}] - Configuration options for the generator
+   */
+  constructor(options: MarkdownGeneratorOptions = {}) {
+    this.dir = options.dir || '.';
+    this.outputFilePath = options.outputFilePath || './prompt.md';
+    this.fileTypeExclusions = new Set(
+      options.fileTypeExclusions || fileTypeExclusions,
+    );
+    this.fileExclusions = options.fileExclusions || [...fileExclusions];
+    // @ts-ignore - options.customPatterns signature is valid
+    this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns);
+    this.verbose = options.verbose !== undefined ? options.verbose : true;
+    this.initialized = false;
+    this.todoPrompt = prompts.getPrompt(options.todoPrompt)
+  }
+
+  /**
+   * Initializes the MarkdownGenerator by loading all nested ignore files.
+   * This is automatically called before any file processing operations.
+   * @async
+   * @returns {Promise<void>}
+   */
+  private async initialize(): Promise<void> {
+    if (!this.initialized) {
+      await this.loadNestedIgnoreFiles();
+      await this.updateGitignore();
+      this.initialized = true;
+    }
+  }
+
+  /**
+   * Loads and processes .toak-ignore files recursively from the project directory.
+   * These files contain patterns for files to exclude from processing.
+   * @async
+   * @returns {Promise<void>}
+   * @throws {Error} When unable to read ignore files
+   */
+  async loadNestedIgnoreFiles(): Promise<void> {
+    try {
+      if (this.verbose) {
+        console.log('Loading ignore patterns...');
+      }
+
+      const ignoreFiles = await glob('**/.toak-ignore', {
+        cwd: this.dir,
+        dot: true,
+        absolute: true,
+        follow: false,
+        nodir: true
+      });
+
+      if (this.verbose) {
+        console.log(`Found ${ignoreFiles.length} ignore files`);
+      }
+
+      // Process each ignore file
+      for (const ignoreFile of ignoreFiles) {
+        try {
+          const content = readFileSync(ignoreFile, 'utf-8');
+          const patterns = content
+            .split('\n')
+            .map(line => line.trim())
+            .filter(line => line && !line.startsWith('#'));
+
+          // Get relative patterns based on ignore file location
+          const ignoreFileDir = path.relative(this.dir, path.dirname(ignoreFile));
+          const relativePatterns = patterns.map(pattern => {
+            if (!pattern.startsWith('/') && !pattern.startsWith('**')) {
+              return path.join(ignoreFileDir, pattern).replace(/\\/g, '/');
+            }
+            return pattern;
+          });
+
+          this.fileExclusions.push(...relativePatterns);
+        } catch (error) {
+          if (this.verbose) {
+            console.error(`Error processing ignore file ${ignoreFile}:`, error);
+          }
+        }
+      }
+
+      // Remove duplicates
+      this.fileExclusions = [...new Set(this.fileExclusions)];
+
+      if (this.verbose) {
+        console.log(`Total exclusion patterns: ${this.fileExclusions.length}`);
+      }
+    } catch (error) {
+      if (this.verbose) {
+        console.error('Error loading nested ignore files:', error);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Retrieves a list of files tracked by Git, excluding those specified in fileTypeExclusions and fileExclusions.
+   * @async
+   * @returns {Promise<string[]>} Array of tracked file paths that aren't excluded
+   * @throws {Error} When unable to execute git command or access files
+   */
+  async getTrackedFiles(): Promise<string[]> {
+    await this.initialize();
+    try {
+      const output = this.execCommand('git ls-files');
+      const trackedFiles = output.split('\n').filter(file => file.trim().length > 0);
+      if (this.verbose) {
+        console.log(`Total tracked files: ${trackedFiles.length}`);
+      }
+      const filteredFiles = trackedFiles.filter(file => {
+        const fileExt = path.extname(file).toLowerCase();
+        return !this.fileTypeExclusions.has(fileExt) && !micromatch.isMatch(file, this.fileExclusions, { dot: true });
+      });
+      if (this.verbose) {
+        const excludedCount = trackedFiles.length - filteredFiles.length;
+        console.log(`Excluded files: ${excludedCount}`);
+        console.log(`Files to process after exclusions: ${filteredFiles.length}`);
+      }
+      return filteredFiles;
+    } catch (error) {
+      if (this.verbose) {
+        console.error('Error fetching tracked files:', error);
+      }
+      return [];
+    }
+  }
+
+  /**
+   * Reads and processes the content of a file, cleaning and redacting sensitive information.
+   * @async
+   * @param {string} filePath - Path to the file to read
+   * @returns {Promise<string>} Cleaned and redacted content of the file
+   * @throws {Error} When unable to read or process the file
+   */
+  async readFileContent(filePath: string): Promise<string> {
+    try {
+      const content = await readFile(filePath, 'utf-8');
+      const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content);
+      if (this.verbose) {
+        const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length;
+        console.log(`${filePath}: Tokens[${tokenCount}]`);
+      }
+      return  cleanedAndRedactedContent.trimEnd();
+    } catch (error) {
+      if (this.verbose) {
+        console.error(`Error reading file ${filePath}:`, error);
+      }
+      return '';
+    }
+  }
+
+  /**
+   * Generates markdown content from all tracked files in the project.
+   * @async
+   * @returns {Promise<string>} Generated markdown content containing all processed files
+   * @throws {Error} When unable to generate markdown content
+   */
+  async generateMarkdown(): Promise<string> {
+    const trackedFiles = await this.getTrackedFiles();
+    if (this.verbose) {
+      console.log(`Generating markdown for ${trackedFiles.length} files`);
+    }
+    let markdownContent = '# Project Files\n\n';
+
+    for (const file of trackedFiles) {
+      const absolutePath = path.join(this.dir, file);
+      const content = await this.readFileContent(absolutePath);
+      if (content.trim()) { // Only include files with content after cleaning
+        markdownContent += `## ${file}\n~~~\n${content.trim()}\n~~~\n\n`;
+      } else if (this.verbose) {
+        console.log(`Skipping ${file} as it has no content after cleaning.`);
+      }
+    }
+    return markdownContent;
+  }
+
+  /**
+   * Retrieves the content of the project's todo file, creating it if it doesn't exist.
+   * @async
+   * @returns {Promise<string>} Content of the todo file
+   * @throws {Error} When unable to read or create the todo file
+   */
+  async getTodo(): Promise<string> {
+    const todoPath = path.join(this.dir, 'todo');
+    try {
+      if (this.verbose) {
+        console.log('Reading todo file');
+      }
+      return await readFile(todoPath, 'utf-8');
+    } catch (error: any) {
+      if (error.code === 'ENOENT') {
+        // File does not exist
+        if (this.verbose) {
+          console.log('File not found, creating a new \'todo\' file.');
+        }
+        await writeFile(todoPath, ''); // Create an empty 'todo' file
+        return await this.getTodo(); // Await the recursive call
+      }
+      if (this.verbose) {
+        console.error('Error reading todo file:', error);
+      }
+      throw error;
+    }
+  }
+
+  async getRootIgnore(): Promise<string> {
+    const rootIgnorePath = path.join(this.dir, '.toak-ignore');
+    try {
+      return await readFile(rootIgnorePath, 'utf-8');
+    } catch (error: any) {
+      if (error.code === 'ENOENT') {
+        // File does not exist
+        if (this.verbose) {
+          console.log('File not found, creating a root \'.toak-ignore\' file.');
+        }
+        await writeFile(rootIgnorePath, 'todo\nprompt.md'); // Create an empty 'todo' file
+        return await this.getRootIgnore(); // Await the recursive call
+      }
+      throw error;
+    }
+  }
+
+  async updateGitignore(): Promise<void> {
+    const gitignorePath = path.join(this.dir, '.gitignore');
+    try {
+      let content = '';
+      try {
+        content = await readFile(gitignorePath, 'utf-8');
+      } catch (error: any) {
+        if (error.code === 'ENOENT') {
+          // .gitignore doesn't exist, create it
+          if (this.verbose) {
+            console.log('File not found, creating a \'.gitignore\' file.');
+          }
+          content = '';
+        } else {
+          throw error;
+        }
+      }
+
+      // Check if entries already exist
+      const lines = content.split('\n');
+      const needsPromptMd = !lines.some(line => line.trim() === 'prompt.md');
+      const needsToakIgnore = !lines.some(line => line.trim() === '.toak-ignore');
+
+      // Add entries if needed
+      if (needsPromptMd || needsToakIgnore) {
+        if (this.verbose) {
+          console.log('Updating .gitignore with prompt.md and .toak-ignore');
+        }
+
+        let newContent = content;
+        if (newContent && !newContent.endsWith('\n')) {
+          newContent += '\n';
+        }
+
+        if (needsPromptMd) {
+          newContent += 'prompt.md\n';
+        }
+
+        if (needsToakIgnore) {
+          newContent += '.toak-ignore\n';
+        }
+
+        await writeFile(gitignorePath, newContent);
+      }
+    } catch (error) {
+      if (this.verbose) {
+        console.error('Error updating .gitignore:', error);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Creates a complete markdown document combining code documentation and todos.
+   * @async
+   * @returns {Promise<Object>} Result object
+   * @returns {boolean} result.success - Whether the operation was successful
+   * @returns {number} [result.tokenCount] - Number of tokens in the generated document
+   * @returns {Error} [result.error] - Error object if operation failed
+   * @throws {Error} When unable to create or write the markdown document
+   */
+  async createMarkdownDocument(): Promise<{ success: boolean, tokenCount?: number, error?: Error }> {
+    try {
+      const codeMarkdown = await this.generateMarkdown();
+      const todos = await this.getTodo();
+      const _ = await this.getRootIgnore();
+      const markdown = codeMarkdown + `\n---\n\n${todos}\n`;
+      await writeFile(this.outputFilePath, markdown);
+      if (this.verbose) {
+        console.log(`Markdown document created at ${this.outputFilePath}`);
+        const totalTokens = llama3Tokenizer.encode(markdown).length;
+        console.log({ total_tokens: totalTokens });
+      }
+      return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length };
+    } catch (error: any) {
+      if (this.verbose) {
+        console.error('Error writing markdown document:', error);
+      }
+      return { success: false, error };
+    }
+  }
+
+  /**
+   * Executes a shell command in the specified directory.
+   * @param {string} command - Shell command to execute
+   * @returns {string} Output of the command
+   * @throws {Error} When command execution fails
+   * @private
+   */
+  private execCommand(command: string): string {
+    try {
+      return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim();
+    } catch (error) {
+      if (this.verbose) {
+        console.error(`Error executing command: ${command}`, error);
+      }
+      throw error;
+    }
+  }
+}

packages/toak/src/TokenCleaner.ts

@@ -0,0 +1,81 @@
+export class TokenCleaner {
+  patterns: { regex: RegExp; replacement: string }[];
+  secretPatterns: { regex: RegExp; replacement: string }[];
+
+  constructor(customPatterns: { regex: RegExp; replacement: string }[] = [], customSecretPatterns: {
+    regex: RegExp;
+    replacement: string
+  }[] = []) {
+    this.patterns = [
+      { regex: /\/\/.*$/gm, replacement: '' }, // Single-line comments
+      { regex: /\/\*[\s\S]*?\*\//g, replacement: '' }, // Multi-line comments
+      { regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, // Console statements
+      { regex: /^\s*[\r\n]/gm, replacement: '' }, // Empty lines
+      { regex: / +$/gm, replacement: '' }, // Trailing spaces
+      { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, // Import statements
+      { regex: /^\s*\n+/gm, replacement: '\n' }, // Multiple newlines
+      ...customPatterns,
+    ];
+    // eslint-no-no-useless-escape
+
+      (this.secretPatterns = [
+        {
+          regex: /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi,
+          replacement: '[REDACTED]',
+        },
+        {
+          regex: /(?<=const\s+\w+\s*=\s*['"])(eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*)(?=['"])/g,
+          replacement: '[REDACTED_JWT]',
+        },
+        {
+          regex: /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi,
+          replacement: '[REDACTED]',
+        },
+        {
+          regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
+          replacement: '[REDACTED]'
+        },
+        {
+          regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
+          replacement: '[REDACTED]',
+        },
+        {
+          regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi,
+          replacement: '[REDACTED_HASH]',
+        },
+        {
+          regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g,
+          replacement: '[REDACTED_BASE64]',
+        },
+        ...customSecretPatterns,
+      ]);
+  }
+
+  clean(code: string): string {
+    return this.patterns.reduce(
+      (cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement),
+      code,
+    );
+  }
+
+  redactSecrets(code: string): string {
+    return this.secretPatterns.reduce(
+      (redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement),
+      code,
+    );
+  }
+
+  cleanAndRedact(code: string): string {
+    // First redact secrets
+    const redactedCode = this.redactSecrets(code);
+
+    // Add pattern to remove lines that only contain redacted content
+    const redactedLines = /^.*\[REDACTED(?:_[A-Z]+)?\].*$/gm;
+    const withoutRedactedLines = redactedCode.replace(redactedLines, '');
+
+    // Then clean the code
+    const cleanedCode = this.clean(withoutRedactedLines);
+
+    return cleanedCode.trim();
+  }
+}

packages/toak/src/cli.ts

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+import type { PresetPrompt } from './prompts';
+
+console.log('RUNNING TOKENIZER');
+import { MarkdownGenerator, type MarkdownGeneratorOptions } from './MarkdownGenerator';
+
+const args = process.argv.slice(2);
+const options: { prompt?: PresetPrompt; } & MarkdownGeneratorOptions = {
+
+};
+type ValidArg = keyof MarkdownGeneratorOptions;
+
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--help') {
+    console.log(`${Object.keys(options).map(item => "--" + item).join(', ')}`);
+  }
+  if (args[i] === '--prompt') {
+    options["todoPrompt"] = args[i + 1]
+    i++;
+  }
+  const arg = args[i].replace(/^--/, '');
+  if (arg as any satisfies ValidArg) {
+      // @ts-ignore - arg can't be used to index options
+    options[arg] = args[i + 1]
+    i++;
+  } else {
+    console.log(`Invalid argument specified: ${arg}`);
+    console.log(`Possible arguments: ${Object.keys(options).map(item => "--" + item).join(', ')}`);
+  }
+}
+
+const generator = new MarkdownGenerator(options);
+generator
+  .createMarkdownDocument()
+  .then((result: { success: boolean }) => {
+    if (!result.success) {
+      process.exit(1);
+    }
+  })
+  .catch((error: any) => {
+    console.error('Error:', error);
+    process.exit(1);
+  });

packages/toak/src/fileExclusions.ts

@@ -0,0 +1,72 @@
+export default [
+  // Config patterns
+  '**/.*rc',
+  '**/.*rc.{js,json,yaml,yml}',
+  '**/*.config.{js,ts}',
+  '**/tsconfig.json',
+  '**/tsconfig*.json',
+  '**/jsconfig.json',
+  '**/jsconfig*.json',
+  '**/package-lock.json',
+  '**/.prettierignore',
+  '**/.dockerignore',
+  // Environment and variables
+  '**/.env*',
+  '**/*.vars',
+  '**/secrets.*',
+
+  // Version control
+  '**/.git*',
+  '**/.hg*',
+  '**/.svn*',
+  '**/CVS',
+  '**/.github/',
+
+  // CI/CD
+  '**/.gitlab-ci.yml',
+  '**/azure-pipelines.yml',
+  '**/jenkins*',
+
+  // Dependency directories
+  '**/node_modules/',
+  '**/target/',
+  '**/__pycache__/',
+  '**/venv/',
+  '**/.venv/',
+  '**/env/',
+  '**/build/',
+  '**/dist/',
+  '**/out/',
+  '**/bin/',
+  '**/obj/',
+
+  // Documentation
+  '**/README*',
+  '**/CHANGELOG*',
+  '**/CONTRIBUTING*',
+  '**/LICENSE*',
+  '**/docs/',
+  '**/documentation/',
+
+  // IDE and editors
+  '**/.{idea,vscode,eclipse,settings,zed,cursor}/',
+  '**/.project',
+  '**/.classpath',
+  '**/.factorypath',
+
+  // Test and data
+  '**/test{s,}/',
+  '**/spec/',
+  '**/fixtures/',
+  '**/testdata/',
+  '**/__tests__/',
+  '**/*.{test,spec}.*',
+  '**/coverage/',
+  '**/jest.config.*',
+
+  // Logs and temporary files
+  '**/logs/',
+  '**/tmp/',
+  '**/temp/',
+  '**/*.log'
+] as const;

packages/toak/src/fileTypeExclusions.ts

@@ -0,0 +1,62 @@
+const filetypeExclusions = [
+  // Images
+  '.jpg',
+  '.jpeg',
+  '.png',
+  '.gif',
+  '.bmp',
+  '.svg',
+  '.webp',
+  '.tiff',
+  '.ico',
+
+  // Fonts
+  '.ttf',
+  '.woff',
+  '.woff2',
+  '.eot',
+  '.otf',
+
+  // Lock files
+  '.lock',
+  '.lockb',
+
+  // Config files
+  // '.yaml',
+  // '.yml',
+  // '.toml',
+  // '.conf',
+
+  // Binary and compiled
+  '.exe',
+  '.dll',
+  '.so',
+  '.dylib',
+  '.bin',
+  '.dat',
+  '.pyc',
+  '.pyo',
+  '.class',
+  '.jar',
+
+  // Archives
+  '.zip',
+  '.tar',
+  '.gz',
+  '.rar',
+  '.7z',
+
+  // Media
+  '.mp3',
+  '.mp4',
+  '.avi',
+  '.mov',
+  '.wav',
+
+  // Database
+  '.db',
+  '.sqlite',
+  '.sqlite3'
+] as const;
+
+export default filetypeExclusions;

packages/toak/src/index.ts

@@ -0,0 +1,2 @@
+export { TokenCleaner } from './TokenCleaner';
+export { MarkdownGenerator } from './MarkdownGenerator';

packages/toak/src/prompts.ts

@@ -0,0 +1,39 @@
+const taskConditionsStandard_FixErrors = `
+## Task
+1. Fix these errors.
+## Conditions
+2. Output labeled and fully fixed files only, no diffs.
+## Standard
+3. Respond with the files, no examples or excessive explanations.
+
+~~~console
+clean up your errors and put them here
+~~~
+`;
+
+
+
+
+
+
+export const customPrompts: Record<string, string> = {
+  "tcs:fix:errors": taskConditionsStandard_FixErrors,
+};
+
+export type PresetPrompt = keyof typeof customPrompts;
+
+
+export function isPreset(key: string): boolean {
+  return key in customPrompts;
+}
+
+
+export const prompts = {
+  ...customPrompts,
+  default: customPrompts["tcs:fix:errors"],
+  getPrompt(key?: string) {
+    if (!key) return prompts.default;
+    if (!isPreset(key)) return prompts.default;
+    return customPrompts[key];
+  }
+};

packages/toak/test-for-ignore.css

@@ -0,0 +1,46 @@
+/* This file is a sample used for testing exclusions */
+body {
+    margin: 0;
+    padding: 0;
+    font-family: Arial, sans-serif;
+    background-color: #f4f4f4;
+    line-height: 1.6;
+}
+
+h1, h2, h3 {
+    color: #333;
+    margin-bottom: 1rem;
+}
+
+p {
+    color: #555;
+    margin-bottom: 1rem;
+}
+
+a {
+    color: #007bff;
+    text-decoration: none;
+}
+
+a:hover {
+    text-decoration: underline;
+}
+
+button {
+    background-color: #007bff;
+    color: white;
+    border: none;
+    padding: 10px 15px;
+    cursor: pointer;
+    border-radius: 5px;
+}
+
+button:hover {
+    background-color: #0056b3;
+}
+
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+    padding: 20px;
+}

packages/toak/test/cli.test.ts

@@ -0,0 +1,100 @@
+// test/cli.test.ts
+import { describe, it, expect, beforeEach, afterEach, spyOn, mock } from 'bun:test';
+import { MarkdownGenerator, type MarkdownGeneratorOptions } from '../src/MarkdownGenerator';
+import type { PresetPrompt } from '../src/prompts';
+
+// Function to process CLI arguments similar to cli.ts
+function processArgs(args: string[]): { prompt?: PresetPrompt; } & MarkdownGeneratorOptions {
+  const options: { prompt?: PresetPrompt; } & MarkdownGeneratorOptions = {};
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--help') {
+      console.log(`${Object.keys(options).map(item => "--" + item).join(', ')}`);
+      continue;
+    }
+    if (args[i] === '--prompt') {
+      options["todoPrompt"] = args[i + 1] as PresetPrompt;
+      i++;
+      continue;
+    }
+
+    const arg = args[i].replace(/^--/, '');
+    if (['dir', 'outputFilePath', 'verbose', 'todoPrompt', 'fileTypeExclusions', 'fileExclusions', 'customPatterns', 'customSecretPatterns'].includes(arg)) {
+      // @ts-ignore - dynamic property access
+      options[arg] = args[i + 1];
+      i++;
+    } else {
+      console.log(`Invalid argument specified: ${arg}`);
+      console.log(`Possible arguments: ${Object.keys(options).map(item => "--" + item).join(', ')}`);
+    }
+  }
+
+  return options;
+}
+
+describe('CLI', () => {
+  describe('argument handling', () => {
+    it('should process --prompt argument correctly', () => {
+      // Set up test arguments
+      const args = ['--prompt', 'tcs:fix:errors'];
+
+      // Process arguments
+      const options = processArgs(args);
+
+      // Verify options
+      expect(options).toEqual(expect.objectContaining({
+        todoPrompt: 'tcs:fix:errors'
+      }));
+    });
+
+    it('should process other valid arguments correctly', () => {
+      // Set up test arguments
+      const args = ['--dir', './src', '--outputFilePath', './custom.md', '--verbose', 'false'];
+
+      // Process arguments
+      const options = processArgs(args);
+
+      // Verify options
+      expect(options).toEqual(expect.objectContaining({
+        dir: './src',
+        outputFilePath: './custom.md',
+        verbose: 'false'
+      }));
+    });
+
+    it('should handle invalid arguments', () => {
+      // Mock console.log to capture output
+      const consoleLogSpy = spyOn(console, 'log');
+
+      // Set up test arguments
+      const args = ['--invalidArg', 'value'];
+
+      // Process arguments
+      processArgs(args);
+
+      // Verify error message was logged
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid argument specified: invalidArg'));
+
+      // Restore original function
+      consoleLogSpy.mockRestore();
+    });
+
+    it('should display help when --help argument is provided', () => {
+      // Mock console.log to capture output
+      const consoleLogSpy = spyOn(console, 'log');
+
+      // Set up test arguments
+      const args = ['--help'];
+
+      // Process arguments
+      processArgs(args);
+
+      // Verify help message was logged
+      // At this point, the options object is empty, so we just check that console.log was called
+      expect(consoleLogSpy).toHaveBeenCalled();
+
+      // Restore original function
+      consoleLogSpy.mockRestore();
+    });
+  });
+});

packages/toak/test/core.test.ts

@@ -0,0 +1,517 @@
+// test/core.test.ts
+import { describe, it, expect, beforeEach, spyOn, mock } from 'bun:test';
+import { TokenCleaner, MarkdownGenerator } from '../src';
+import * as micromatch from 'micromatch';
+import llama3Tokenizer from 'llama3-tokenizer-js';
+import path from 'path';
+import * as fs from 'fs/promises';
+import * as child_process from 'child_process';
+import { writeFile } from 'fs/promises';
+
+
+describe('TokenCleaner', () => {
+  let tokenCleaner: TokenCleaner;
+
+  beforeEach(() => {
+    tokenCleaner = new TokenCleaner();
+  });
+
+  describe('clean', () => {
+    it('should remove single-line comments', () => {
+      const code = `const a = 1; // This is a comment
+const b = 2;`;
+      const expected = `const a = 1;
+const b = 2;`;
+      expect(tokenCleaner.clean(code)).toBe(expected);
+    });
+
+    it('should remove multi-line comments', () => {
+      const code = `/* This is a 
+multi-line comment */
+const a = 1;`;
+      const expected = `const a = 1;`;
+      expect(tokenCleaner.clean(code)).toBe(expected);
+    });
+
+    it('should remove console statements', () => {
+      const code = `console.log('Debugging');
+const a = 1;`;
+      const expected = `const a = 1;`;
+      expect(tokenCleaner.clean(code)).toBe(expected);
+    });
+
+    it('should remove import statements', () => {
+      const code = `import fs from 'fs';
+const a = 1;`;
+      const expected = `
+const a = 1;`;
+      expect(tokenCleaner.clean(code)).toBe(expected);
+    });
+
+    it('should trim whitespace and empty lines', () => {
+      const code = `const a = 1;  
+
+
+const b = 2;  `;
+      const expected = `const a = 1;
+const b = 2;`;
+      expect(tokenCleaner.clean(code)).toBe(expected);
+    });
+
+    it('should apply custom patterns', () => {
+      const customPatterns = [
+        { regex: /DEBUG\s*=\s*true/g, replacement: 'DEBUG = false' },
+      ];
+      const customTokenCleaner = new TokenCleaner(customPatterns);
+      const code = `const DEBUG = true;
+const a = 1;`;
+      const expected = `const DEBUG = false;
+const a = 1;`;
+      expect(customTokenCleaner.clean(code)).toBe(expected);
+    });
+  });
+
+  describe('redactSecrets', () => {
+    it('should redact API keys', () => {
+      const code = `const apiKey = '12345-ABCDE';`;
+      const expected = `const apiKey = '[REDACTED]';`;
+      expect(tokenCleaner.redactSecrets(code)).toBe(expected);
+    });
+
+    it('should redact bearer tokens', () => {
+      const code = `Authorization: Bearer abcdef123456`;
+      const expected = `Authorization: Bearer [REDACTED]`;
+      expect(tokenCleaner.redactSecrets(code)).toBe(expected);
+    });
+
+    it('should redact JWT tokens', () => {
+      const code = `const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.XmX8v1';`;
+      const expected = `const token = '[REDACTED_JWT]';`;
+      expect(tokenCleaner.redactSecrets(code)).toBe(expected);
+    });
+
+    it('should redact hashes', () => {
+      const code = `const hash = 'abcdef1234567890abcdef1234567890abcdef12';`;
+      const expected = `const hash = '[REDACTED_HASH]';`;
+      expect(tokenCleaner.redactSecrets(code)).toBe(expected);
+    });
+
+    it('should apply custom secret patterns', () => {
+      const customSecretPatterns = [
+        { regex: /SECRET_KEY:\s*['"]([^'"]+)['"]/g, replacement: 'SECRET_KEY: [REDACTED]' },
+      ];
+      const customTokenCleaner = new TokenCleaner([], customSecretPatterns);
+      const code = `SECRET_KEY: 'mysecretkey123'`;
+      const expected = `SECRET_KEY: [REDACTED]`;
+      expect(customTokenCleaner.redactSecrets(code)).toBe(expected);
+    });
+  });
+
+  describe('cleanAndRedact', () => {
+    it('should clean and redact code', () => {
+      const code = `// Comment
+const apiKey = '12345-ABCDE';
+console.log('Debugging');
+import fs from 'fs';
+
+/* Multi-line comment */
+const a = 1;`;
+      const expected = `const a = 1;`;
+      expect(tokenCleaner.cleanAndRedact(code)).toBe(expected);
+    });
+
+    it('should handle empty input', () => {
+      const code = ``;
+      expect(tokenCleaner.cleanAndRedact(code)).toBe('');
+    });
+  });
+});
+
+describe('MarkdownGenerator', () => {
+  let markdownGenerator: MarkdownGenerator;
+
+  beforeEach(() => {
+    markdownGenerator = new MarkdownGenerator({ verbose: false });
+  })
+
+  describe('getTrackedFiles', () => {
+    it("should return filtered tracked files", async () => {
+      const mockFiles = ["src/index.ts", "src/MarkdownGenerator.ts", "src/TokenCleaner.ts"];
+
+      // Use Bun's mock instead of Jest's spyOn
+      mock.module("child_process", () => ({
+        execSync: () => mockFiles.join('\n')
+      }));
+
+      // Mock micromatch using Bun's mock
+      mock.module("micromatch", () => ({
+        isMatch: () => false
+      }));
+
+      const trackedFiles = await markdownGenerator.getTrackedFiles();
+      expect(trackedFiles).toEqual(mockFiles);
+    });
+
+    it('should handle git command failure', async () => {
+      // Spy on execSync to throw an error
+      const execSyncSpy = spyOn(child_process, 'execSync').mockImplementation(() => {
+        throw new Error('Git command failed');
+      });
+
+      const trackedFiles = await markdownGenerator.getTrackedFiles();
+      expect(execSyncSpy).toHaveBeenCalled();
+      expect(trackedFiles).toEqual([]);
+
+      // Restore the original implementation
+      execSyncSpy.mockRestore();
+    });
+  });
+
+  describe('readFileContent', () => {
+    it("should read and clean file content", async () => {
+      const filePath = "test.ts";
+      const rawContent = "// comment\nconst x = 1;\nconsole.log('test');";
+      const cleanedContent = "const x = 1;";
+
+      // Mock fs/promises readFile
+      mock.module("fs/promises", () => ({
+        readFile: async () => rawContent,
+        writeFile: async () => {
+        }
+      }));
+
+      // Mock TokenCleaner
+      const cleanerMock = mock(() => cleanedContent);
+      TokenCleaner.prototype.cleanAndRedact = cleanerMock;
+
+      // Mock llama3Tokenizer
+      mock.module("llama3-tokenizer-js", () => ({
+        encode: () => [1, 2, 3]
+      }));
+
+      const content = await markdownGenerator.readFileContent(filePath);
+      expect(content).toBe(cleanedContent);
+      expect(cleanerMock).toHaveBeenCalled();
+    });
+
+    it('should handle readFile failure', async () => {
+      const filePath = 'src/missing.ts';
+
+      // Spy on fs.readFile to reject
+      const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue(new Error('File not found'));
+
+      const content = await markdownGenerator.readFileContent(filePath);
+      expect(readFileSpy).toHaveBeenCalledWith(filePath, 'utf-8');
+      expect(content).toBe('');
+
+      // Restore the original implementation
+      readFileSpy.mockRestore();
+    });
+  });
+
+  describe('generateMarkdown', () => {
+    it('should generate markdown content from tracked files', async () => {
+      // Spy on getTrackedFiles
+      const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([
+        'src/index.ts',
+        'src/MarkdownGenerator.ts',
+      ]);
+
+      // Spy on readFileContent
+      const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => {
+        if (filePath === path.join('.', 'src/index.ts')) {
+          return `const a = 1;`;
+        } else if (filePath === path.join('.', 'src/MarkdownGenerator.ts')) {
+          return `class MarkdownGenerator {}`;
+        }
+        return '';
+      });
+
+      const expectedMarkdown = `# Project Files
+
+## src/index.ts
+~~~
+const a = 1;
+~~~
+
+## src/MarkdownGenerator.ts
+~~~
+class MarkdownGenerator {}
+~~~
+
+`;
+
+      const markdown = await markdownGenerator.generateMarkdown();
+      expect(markdown).toBe(expectedMarkdown);
+
+      // Restore the original implementations
+      getTrackedFilesSpy.mockRestore();
+      readFileContentSpy.mockRestore();
+    });
+
+    it('should handle no tracked files', async () => {
+      // Spy on getTrackedFiles
+      const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([]);
+
+      const expectedMarkdown = `# Project Files
+
+`;
+
+      const markdown = await markdownGenerator.generateMarkdown();
+      expect(markdown).toBe(expectedMarkdown);
+
+      // Restore the original implementation
+      getTrackedFilesSpy.mockRestore();
+    });
+
+    it('should skip empty file contents', async () => {
+      // Spy on getTrackedFiles
+      const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([
+        'src/index.ts',
+        'src/empty.ts',
+      ]);
+
+      // Spy on readFileContent
+      const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => {
+        if (filePath === path.join('.', 'src/index.ts')) {
+          return `const a = 1;`;
+        } else if (filePath === path.join('.', 'src/empty.ts')) {
+          return `   `;
+        }
+        return '';
+      });
+
+      const expectedMarkdown = `# Project Files
+
+## src/index.ts
+~~~
+const a = 1;
+~~~
+
+`;
+
+      const markdown = await markdownGenerator.generateMarkdown();
+      expect(markdown).toBe(expectedMarkdown);
+
+      // Restore the original implementations
+      getTrackedFilesSpy.mockRestore();
+      readFileContentSpy.mockRestore();
+    });
+  });
+
+  describe('getTodo', () => {
+    it('should read the todo file content', async () => {
+      const todoContent = `- [ ] Implement feature X
+- [ ] Fix bug Y`;
+
+      // Spy on fs.readFile
+      const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue(todoContent);
+
+      const todo = await markdownGenerator.getTodo();
+      expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8');
+      expect(todo).toBe(todoContent);
+
+      // Restore the original implementation
+      readFileSpy.mockRestore();
+    });
+
+    it('should create todo file if it does not exist', async () => {
+      const todoPath = path.join('.', 'todo');
+
+      // First call to readFile throws ENOENT, second call resolves to empty string
+      const readFileSpy = spyOn(fs, 'readFile')
+        .mockImplementationOnce(() => {
+          const error: any = new Error('File not found');
+          error.code = 'ENOENT';
+          return Promise.reject(error);
+        })
+        .mockResolvedValueOnce('');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      const todo = await markdownGenerator.getTodo();
+      expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8');
+      expect(writeFileSpy).toHaveBeenCalledWith(todoPath, '');
+      expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8');
+      expect(todo).toBe('');
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+
+    it('should throw error for non-ENOENT errors', async () => {
+      // Spy on fs.readFile to reject with a different error
+      const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue({ code: 'EACCES' });
+
+      await expect(markdownGenerator.getTodo()).rejects.toEqual({ code: 'EACCES' });
+      expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8');
+
+      // Restore the original implementation
+      readFileSpy.mockRestore();
+    });
+  });
+
+  describe('getRootIgnore', () => {
+
+    it('should create root ignore file if it does not exist', async () => {
+      const rootIgnorePath = path.join('.', '.toak-ignore');
+
+      // First call to readFile throws ENOENT, second call resolves to empty string
+      const readFileSpy = spyOn(fs, 'readFile')
+        .mockImplementationOnce(() => {
+          const error: any = new Error('File not found');
+          error.code = 'ENOENT';
+          return Promise.reject(error);
+        })
+        .mockResolvedValueOnce('');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      const rootIgnore = await markdownGenerator.getRootIgnore();
+      expect(readFileSpy).toHaveBeenCalledWith(rootIgnorePath, 'utf-8');
+      expect(writeFileSpy).toHaveBeenCalledWith(rootIgnorePath, 'todo\nprompt.md');
+      expect(rootIgnore).toBe('');
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+  });
+
+  describe('updateGitignore', () => {
+    it('should update .gitignore with prompt.md and .toak-ignore on first run', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to simulate .gitignore exists but doesn't have the entries
+      const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue('node_modules\ndist\n');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was called with correct content
+      expect(writeFileSpy).toHaveBeenCalledWith(
+        gitignorePath, 
+        'node_modules\ndist\nprompt.md\n.toak-ignore\n'
+      );
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+
+    it('should not update .gitignore if entries already exist', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to simulate .gitignore already has the entries
+      const readFileSpy = spyOn(fs, 'readFile')
+        .mockResolvedValue('node_modules\ndist\nprompt.md\n.toak-ignore\n');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was NOT called
+      expect(writeFileSpy).not.toHaveBeenCalled();
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+
+    it('should create .gitignore if it does not exist', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to throw ENOENT error
+      const readFileSpy = spyOn(fs, 'readFile').mockImplementation(() => {
+        const error: any = new Error('File not found');
+        error.code = 'ENOENT';
+        return Promise.reject(error);
+      });
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was called with correct content
+      expect(writeFileSpy).toHaveBeenCalledWith(
+        gitignorePath, 
+        'prompt.md\n.toak-ignore\n'
+      );
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+  });
+
+  describe('createMarkdownDocument', () => {
+    it('should create markdown document successfully', async () => {
+      const mockContent = '# Project Files\n\n## test.txt\n~~~\ntest\n~~~\n\n';
+      const mockTodo = 'test todo';
+      let writeFileCalled = false;
+
+      // Create instance first
+      const generator = new MarkdownGenerator();
+
+      // Setup instance method mocks
+      generator.generateMarkdown = mock(() => Promise.resolve(mockContent));
+      generator.getTodo = mock(() => Promise.resolve(mockTodo));
+
+      // Create a mock implementation for createMarkdownDocument that skips file writing
+      const originalCreateMarkdown = generator.createMarkdownDocument.bind(generator);
+      generator.createMarkdownDocument = mock(async () => {
+        writeFileCalled = true;
+        const markdown = await generator.generateMarkdown();
+        const todos = await generator.getTodo();
+        const fullMarkdown = markdown + `\n---\n\n${todos}\n`;
+        return {
+          success: true,
+          tokenCount: llama3Tokenizer.encode(fullMarkdown).length
+        };
+      });
+
+      // Mock tokenizer with actual observed token count from logs
+      mock(llama3Tokenizer, 'encode').mockImplementation(() => new Array(21));
+
+      const result = await generator.createMarkdownDocument();
+
+      expect(generator.generateMarkdown).toHaveBeenCalled();
+      expect(generator.getTodo).toHaveBeenCalled();
+      expect(writeFileCalled).toBe(true);
+      expect(result.success).toBe(true);
+      expect(result.tokenCount).toBe(21);
+
+    });
+
+    it('should handle errors during markdown creation', async () => {
+      // Spy on generateMarkdown to reject
+      const generateMarkdownSpy = spyOn(markdownGenerator, 'generateMarkdown').mockRejectedValue(new Error('Generation failed'));
+
+      const result = await markdownGenerator.createMarkdownDocument();
+      expect(result.success).toBe(false);
+      expect(result.error).toEqual(new Error('Generation failed'));
+
+      // Restore the original implementation
+      generateMarkdownSpy.mockRestore();
+    });
+  });
+});

packages/toak/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}