wip

Enhanced MarkdownGenerator to update .gitignore programmatically when required, introduced tests for updateGitignore functionality, and ensured `.toak-ignore` is added.

.gitignore

@@ -3,3 +3,5 @@
 /dist/
 prompt.md
 todo
+.toak-ignore
+/google_gemma-3-1b-it-Q6_K.llamafile

package.json

@@ -45,7 +45,9 @@
     "lint:fix": "eslint src/ --fix",
     "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md,yml,yaml}\"",
     "fix": "bun format && bun lint:fix",
-    "release": "bunx release-it"
+    "release": "bunx release-it",
+    "run:inference": "./google_gemma-3-1b-it-Q6_K.llamafile --server --nobrowser",
+    "download:inference": "./download_optimization_model.sh"
   },
   "dependencies": {
     "glob": "^11.0.1",

scripts/download_optimization_model.sh

@@ -0,0 +1,4 @@
+MODEL_URL=https://huggingface.co/Mozilla/gemma-3-1b-it-llamafile/resolve/main/google_gemma-3-1b-it-Q6_K.llamafile?download=true
+
+wget "${MODEL_URL}"
+

scripts/optimize-ignore.ts

@@ -0,0 +1,30 @@
+export async function optimizeToakIgnore(content: string) {
+  const inferenceProcess = Bun.spawn(['bun', 'run:inference']);
+
+  await new Promise(resolve => setTimeout(resolve, 5000));
+
+  const prompt = `You are a helpful assistant.
+## Context
+~~~
+${content}
+~~~
+Respond with a list of files that should be added to the .toak-ignore file to reduce noise in the context. No extra text or explanations.`;
+
+  async function run() {
+    try {
+      const response = await fetch('http://127.0.0.1:8080/completion', {
+        method: 'POST',
+        body: JSON.stringify({
+          prompt,
+          n_predict: 512,
+        }),
+      });
+      const data = await response.json();
+      console.log(data.content);
+    } catch (error) {
+      console.error('Error:', error);
+    }
+  }
+  await run();
+  inferenceProcess.kill();
+}

src/MarkdownGenerator.ts

@@ -64,6 +64,7 @@ export class MarkdownGenerator {
   private async initialize(): Promise<void> {
     if (!this.initialized) {
       await this.loadNestedIgnoreFiles();
+      await this.updateGitignore();
       this.initialized = true;
     }
   }
@@ -260,6 +261,58 @@ export class MarkdownGenerator {
     }
   }
 
+  async updateGitignore(): Promise<void> {
+    const gitignorePath = path.join(this.dir, '.gitignore');
+    try {
+      let content = '';
+      try {
+        content = await readFile(gitignorePath, 'utf-8');
+      } catch (error: any) {
+        if (error.code === 'ENOENT') {
+          // .gitignore doesn't exist, create it
+          if (this.verbose) {
+            console.log('File not found, creating a \'.gitignore\' file.');
+          }
+          content = '';
+        } else {
+          throw error;
+        }
+      }
+
+      // Check if entries already exist
+      const lines = content.split('\n');
+      const needsPromptMd = !lines.some(line => line.trim() === 'prompt.md');
+      const needsToakIgnore = !lines.some(line => line.trim() === '.toak-ignore');
+
+      // Add entries if needed
+      if (needsPromptMd || needsToakIgnore) {
+        if (this.verbose) {
+          console.log('Updating .gitignore with prompt.md and .toak-ignore');
+        }
+
+        let newContent = content;
+        if (newContent && !newContent.endsWith('\n')) {
+          newContent += '\n';
+        }
+
+        if (needsPromptMd) {
+          newContent += 'prompt.md\n';
+        }
+
+        if (needsToakIgnore) {
+          newContent += '.toak-ignore\n';
+        }
+
+        await writeFile(gitignorePath, newContent);
+      }
+    } catch (error) {
+      if (this.verbose) {
+        console.error('Error updating .gitignore:', error);
+      }
+      throw error;
+    }
+  }
+
   /**
    * Creates a complete markdown document combining code documentation and todos.
    * @async

src/cli.ts

@@ -3,10 +3,22 @@ import type { PresetPrompt } from './prompts';
 
 console.log('RUNNING TOKENIZER');
 import { MarkdownGenerator, type MarkdownGeneratorOptions } from './MarkdownGenerator';
+import { optimizeToakIgnore } from '../scripts/optimize-ignore.ts';
+
+let output = '';
+process.stdout.write = (write => {
+  return (str: string | Uint8Array, ...args: any) => {
+    output += str.toString();
+    return write.apply(process.stdout, [str, ...args]);
+  };
+})(process.stdout.write);
+
+let optimizeIgnore = false;
 
 const args = process.argv.slice(2);
 const options: { prompt?: PresetPrompt; } & MarkdownGeneratorOptions = {
 
+
 };
 type ValidArg = keyof MarkdownGeneratorOptions;
 
@@ -18,6 +30,10 @@ for (let i = 0; i < args.length; i++) {
     options["todoPrompt"] = args[i + 1]
     i++;
   }
+  if (args[i] === '--optimize-ignore') {
+    optimizeIgnore = true;
+    i++;
+  }
   const arg = args[i].replace(/^--/, '');
   if (arg as any satisfies ValidArg) {
       // @ts-ignore - arg can't be used to index options
@@ -30,12 +46,19 @@ for (let i = 0; i < args.length; i++) {
 }
 
 const generator = new MarkdownGenerator(options);
+
+
+
+
 generator
   .createMarkdownDocument()
-  .then((result: { success: boolean }) => {
+  .then(async (result: { success: boolean }) => {
     if (!result.success) {
       process.exit(1);
     }
+    if (optimizeIgnore) {
+      await optimizeToakIgnore(output);
+    }
   })
   .catch((error: any) => {
     console.error('Error:', error);

test/core.test.ts

@@ -50,8 +50,8 @@ const a = 1;`;
 
     it('should trim whitespace and empty lines', () => {
       const code = `const a = 1;  
-    
-    
+
+
 const b = 2;  `;
       const expected = `const a = 1;
 const b = 2;`;
@@ -381,6 +381,88 @@ const a = 1;
     });
   });
 
+  describe('updateGitignore', () => {
+    it('should update .gitignore with prompt.md and .toak-ignore on first run', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to simulate .gitignore exists but doesn't have the entries
+      const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue('node_modules\ndist\n');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was called with correct content
+      expect(writeFileSpy).toHaveBeenCalledWith(
+        gitignorePath, 
+        'node_modules\ndist\nprompt.md\n.toak-ignore\n'
+      );
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+
+    it('should not update .gitignore if entries already exist', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to simulate .gitignore already has the entries
+      const readFileSpy = spyOn(fs, 'readFile')
+        .mockResolvedValue('node_modules\ndist\nprompt.md\n.toak-ignore\n');
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was NOT called
+      expect(writeFileSpy).not.toHaveBeenCalled();
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+
+    it('should create .gitignore if it does not exist', async () => {
+      const gitignorePath = path.join('.', '.gitignore');
+
+      // Mock readFile to throw ENOENT error
+      const readFileSpy = spyOn(fs, 'readFile').mockImplementation(() => {
+        const error: any = new Error('File not found');
+        error.code = 'ENOENT';
+        return Promise.reject(error);
+      });
+
+      // Spy on fs.writeFile
+      const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined);
+
+      // Call the method
+      await markdownGenerator.updateGitignore();
+
+      // Verify readFile was called
+      expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8');
+
+      // Verify writeFile was called with correct content
+      expect(writeFileSpy).toHaveBeenCalledWith(
+        gitignorePath, 
+        'prompt.md\n.toak-ignore\n'
+      );
+
+      // Restore the original implementations
+      readFileSpy.mockRestore();
+      writeFileSpy.mockRestore();
+    });
+  });
+
   describe('createMarkdownDocument', () => {
     it('should create markdown document successfully', async () => {
       const mockContent = '# Project Files\n\n## test.txt\n~~~\ntest\n~~~\n\n';