// test/core.test.ts import { describe, it, expect, beforeEach, spyOn, mock } from 'bun:test'; import { TokenCleaner, MarkdownGenerator } from '../src'; import * as micromatch from 'micromatch'; import llama3Tokenizer from 'llama3-tokenizer-js'; import path from 'path'; import * as fs from 'fs/promises'; import * as child_process from 'child_process'; import { writeFile } from 'fs/promises'; describe('TokenCleaner', () => { let tokenCleaner: TokenCleaner; beforeEach(() => { tokenCleaner = new TokenCleaner(); }); describe('clean', () => { it('should remove single-line comments', () => { const code = `const a = 1; // This is a comment const b = 2;`; const expected = `const a = 1; const b = 2;`; expect(tokenCleaner.clean(code)).toBe(expected); }); it('should remove multi-line comments', () => { const code = `/* This is a multi-line comment */ const a = 1;`; const expected = `const a = 1;`; expect(tokenCleaner.clean(code)).toBe(expected); }); it('should remove console statements', () => { const code = `console.log('Debugging'); const a = 1;`; const expected = `const a = 1;`; expect(tokenCleaner.clean(code)).toBe(expected); }); it('should remove import statements', () => { const code = `import fs from 'fs'; const a = 1;`; const expected = ` const a = 1;`; expect(tokenCleaner.clean(code)).toBe(expected); }); it('should trim whitespace and empty lines', () => { const code = `const a = 1; const b = 2; `; const expected = `const a = 1; const b = 2;`; expect(tokenCleaner.clean(code)).toBe(expected); }); it('should apply custom patterns', () => { const customPatterns = [ { regex: /DEBUG\s*=\s*true/g, replacement: 'DEBUG = false' }, ]; const customTokenCleaner = new TokenCleaner(customPatterns); const code = `const DEBUG = true; const a = 1;`; const expected = `const DEBUG = false; const a = 1;`; expect(customTokenCleaner.clean(code)).toBe(expected); }); }); describe('redactSecrets', () => { it('should redact API keys', () => { const code = `const apiKey = '12345-ABCDE';`; const expected = `const apiKey = '[REDACTED]';`; expect(tokenCleaner.redactSecrets(code)).toBe(expected); }); it('should redact bearer tokens', () => { const code = `Authorization: Bearer abcdef123456`; const expected = `Authorization: Bearer [REDACTED]`; expect(tokenCleaner.redactSecrets(code)).toBe(expected); }); it('should redact JWT tokens', () => { const code = `const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.XmX8v1';`; const expected = `const token = '[REDACTED_JWT]';`; expect(tokenCleaner.redactSecrets(code)).toBe(expected); }); it('should redact hashes', () => { const code = `const hash = 'abcdef1234567890abcdef1234567890abcdef12';`; const expected = `const hash = '[REDACTED_HASH]';`; expect(tokenCleaner.redactSecrets(code)).toBe(expected); }); it('should apply custom secret patterns', () => { const customSecretPatterns = [ { regex: /SECRET_KEY:\s*['"]([^'"]+)['"]/g, replacement: 'SECRET_KEY: [REDACTED]' }, ]; const customTokenCleaner = new TokenCleaner([], customSecretPatterns); const code = `SECRET_KEY: 'mysecretkey123'`; const expected = `SECRET_KEY: [REDACTED]`; expect(customTokenCleaner.redactSecrets(code)).toBe(expected); }); }); describe('cleanAndRedact', () => { it('should clean and redact code', () => { const code = `// Comment const apiKey = '12345-ABCDE'; console.log('Debugging'); import fs from 'fs'; /* Multi-line comment */ const a = 1;`; const expected = `const a = 1;`; expect(tokenCleaner.cleanAndRedact(code)).toBe(expected); }); it('should handle empty input', () => { const code = ``; expect(tokenCleaner.cleanAndRedact(code)).toBe(''); }); }); }); describe('MarkdownGenerator', () => { let markdownGenerator: MarkdownGenerator; beforeEach(() => { markdownGenerator = new MarkdownGenerator({ verbose: false }); }) describe('getTrackedFiles', () => { it("should return filtered tracked files", async () => { const mockFiles = ["src/index.ts", "src/MarkdownGenerator.ts", "src/TokenCleaner.ts"]; // Use Bun's mock instead of Jest's spyOn mock.module("child_process", () => ({ execSync: () => mockFiles.join('\n') })); // Mock micromatch using Bun's mock mock.module("micromatch", () => ({ isMatch: () => false })); const trackedFiles = await markdownGenerator.getTrackedFiles(); expect(trackedFiles).toEqual(mockFiles); }); it('should handle git command failure', async () => { // Spy on execSync to throw an error const execSyncSpy = spyOn(child_process, 'execSync').mockImplementation(() => { throw new Error('Git command failed'); }); const trackedFiles = await markdownGenerator.getTrackedFiles(); expect(execSyncSpy).toHaveBeenCalled(); expect(trackedFiles).toEqual([]); // Restore the original implementation execSyncSpy.mockRestore(); }); }); describe('readFileContent', () => { it("should read and clean file content", async () => { const filePath = "test.ts"; const rawContent = "// comment\nconst x = 1;\nconsole.log('test');"; const cleanedContent = "const x = 1;"; // Mock fs/promises readFile mock.module("fs/promises", () => ({ readFile: async () => rawContent, writeFile: async () => { } })); // Mock TokenCleaner const cleanerMock = mock(() => cleanedContent); TokenCleaner.prototype.cleanAndRedact = cleanerMock; // Mock llama3Tokenizer mock.module("llama3-tokenizer-js", () => ({ encode: () => [1, 2, 3] })); const content = await markdownGenerator.readFileContent(filePath); expect(content).toBe(cleanedContent); expect(cleanerMock).toHaveBeenCalled(); }); it('should handle readFile failure', async () => { const filePath = 'src/missing.ts'; // Spy on fs.readFile to reject const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue(new Error('File not found')); const content = await markdownGenerator.readFileContent(filePath); expect(readFileSpy).toHaveBeenCalledWith(filePath, 'utf-8'); expect(content).toBe(''); // Restore the original implementation readFileSpy.mockRestore(); }); }); describe('generateMarkdown', () => { it('should generate markdown content from tracked files', async () => { // Spy on getTrackedFiles const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([ 'src/index.ts', 'src/MarkdownGenerator.ts', ]); // Spy on readFileContent const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => { if (filePath === path.join('.', 'src/index.ts')) { return `const a = 1;`; } else if (filePath === path.join('.', 'src/MarkdownGenerator.ts')) { return `class MarkdownGenerator {}`; } return ''; }); const expectedMarkdown = `# Project Files ## src/index.ts ~~~ const a = 1; ~~~ ## src/MarkdownGenerator.ts ~~~ class MarkdownGenerator {} ~~~ `; const markdown = await markdownGenerator.generateMarkdown(); expect(markdown).toBe(expectedMarkdown); // Restore the original implementations getTrackedFilesSpy.mockRestore(); readFileContentSpy.mockRestore(); }); it('should handle no tracked files', async () => { // Spy on getTrackedFiles const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([]); const expectedMarkdown = `# Project Files `; const markdown = await markdownGenerator.generateMarkdown(); expect(markdown).toBe(expectedMarkdown); // Restore the original implementation getTrackedFilesSpy.mockRestore(); }); it('should skip empty file contents', async () => { // Spy on getTrackedFiles const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([ 'src/index.ts', 'src/empty.ts', ]); // Spy on readFileContent const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => { if (filePath === path.join('.', 'src/index.ts')) { return `const a = 1;`; } else if (filePath === path.join('.', 'src/empty.ts')) { return ` `; } return ''; }); const expectedMarkdown = `# Project Files ## src/index.ts ~~~ const a = 1; ~~~ `; const markdown = await markdownGenerator.generateMarkdown(); expect(markdown).toBe(expectedMarkdown); // Restore the original implementations getTrackedFilesSpy.mockRestore(); readFileContentSpy.mockRestore(); }); }); describe('getTodo', () => { it('should read the todo file content', async () => { const todoContent = `- [ ] Implement feature X - [ ] Fix bug Y`; // Spy on fs.readFile const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue(todoContent); const todo = await markdownGenerator.getTodo(); expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8'); expect(todo).toBe(todoContent); // Restore the original implementation readFileSpy.mockRestore(); }); it('should create todo file if it does not exist', async () => { const todoPath = path.join('.', 'todo'); // First call to readFile throws ENOENT, second call resolves to empty string const readFileSpy = spyOn(fs, 'readFile') .mockImplementationOnce(() => { const error: any = new Error('File not found'); error.code = 'ENOENT'; return Promise.reject(error); }) .mockResolvedValueOnce(''); // Spy on fs.writeFile const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); const todo = await markdownGenerator.getTodo(); expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8'); expect(writeFileSpy).toHaveBeenCalledWith(todoPath, ''); expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8'); expect(todo).toBe(''); // Restore the original implementations readFileSpy.mockRestore(); writeFileSpy.mockRestore(); }); it('should throw error for non-ENOENT errors', async () => { // Spy on fs.readFile to reject with a different error const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue({ code: 'EACCES' }); await expect(markdownGenerator.getTodo()).rejects.toEqual({ code: 'EACCES' }); expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8'); // Restore the original implementation readFileSpy.mockRestore(); }); }); describe('getRootIgnore', () => { it('should create root ignore file if it does not exist', async () => { const rootIgnorePath = path.join('.', '.toak-ignore'); // First call to readFile throws ENOENT, second call resolves to empty string const readFileSpy = spyOn(fs, 'readFile') .mockImplementationOnce(() => { const error: any = new Error('File not found'); error.code = 'ENOENT'; return Promise.reject(error); }) .mockResolvedValueOnce(''); // Spy on fs.writeFile const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); const rootIgnore = await markdownGenerator.getRootIgnore(); expect(readFileSpy).toHaveBeenCalledWith(rootIgnorePath, 'utf-8'); expect(writeFileSpy).toHaveBeenCalledWith(rootIgnorePath, 'todo\nprompt.md'); expect(rootIgnore).toBe(''); // Restore the original implementations readFileSpy.mockRestore(); writeFileSpy.mockRestore(); }); }); describe('updateGitignore', () => { it('should update .gitignore with prompt.md and .toak-ignore on first run', async () => { const gitignorePath = path.join('.', '.gitignore'); // Mock readFile to simulate .gitignore exists but doesn't have the entries const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue('node_modules\ndist\n'); // Spy on fs.writeFile const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); // Call the method await markdownGenerator.updateGitignore(); // Verify readFile was called expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8'); // Verify writeFile was called with correct content expect(writeFileSpy).toHaveBeenCalledWith( gitignorePath, 'node_modules\ndist\nprompt.md\n.toak-ignore\n' ); // Restore the original implementations readFileSpy.mockRestore(); writeFileSpy.mockRestore(); }); it('should not update .gitignore if entries already exist', async () => { const gitignorePath = path.join('.', '.gitignore'); // Mock readFile to simulate .gitignore already has the entries const readFileSpy = spyOn(fs, 'readFile') .mockResolvedValue('node_modules\ndist\nprompt.md\n.toak-ignore\n'); // Spy on fs.writeFile const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); // Call the method await markdownGenerator.updateGitignore(); // Verify readFile was called expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8'); // Verify writeFile was NOT called expect(writeFileSpy).not.toHaveBeenCalled(); // Restore the original implementations readFileSpy.mockRestore(); writeFileSpy.mockRestore(); }); it('should create .gitignore if it does not exist', async () => { const gitignorePath = path.join('.', '.gitignore'); // Mock readFile to throw ENOENT error const readFileSpy = spyOn(fs, 'readFile').mockImplementation(() => { const error: any = new Error('File not found'); error.code = 'ENOENT'; return Promise.reject(error); }); // Spy on fs.writeFile const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); // Call the method await markdownGenerator.updateGitignore(); // Verify readFile was called expect(readFileSpy).toHaveBeenCalledWith(gitignorePath, 'utf-8'); // Verify writeFile was called with correct content expect(writeFileSpy).toHaveBeenCalledWith( gitignorePath, 'prompt.md\n.toak-ignore\n' ); // Restore the original implementations readFileSpy.mockRestore(); writeFileSpy.mockRestore(); }); }); describe('createMarkdownDocument', () => { it('should create markdown document successfully', async () => { const mockContent = '# Project Files\n\n## test.txt\n~~~\ntest\n~~~\n\n'; const mockTodo = 'test todo'; let writeFileCalled = false; // Create instance first const generator = new MarkdownGenerator(); // Setup instance method mocks generator.generateMarkdown = mock(() => Promise.resolve(mockContent)); generator.getTodo = mock(() => Promise.resolve(mockTodo)); // Create a mock implementation for createMarkdownDocument that skips file writing const originalCreateMarkdown = generator.createMarkdownDocument.bind(generator); generator.createMarkdownDocument = mock(async () => { writeFileCalled = true; const markdown = await generator.generateMarkdown(); const todos = await generator.getTodo(); const fullMarkdown = markdown + `\n---\n\n${todos}\n`; return { success: true, tokenCount: llama3Tokenizer.encode(fullMarkdown).length }; }); // Mock tokenizer with actual observed token count from logs mock(llama3Tokenizer, 'encode').mockImplementation(() => new Array(21)); const result = await generator.createMarkdownDocument(); expect(generator.generateMarkdown).toHaveBeenCalled(); expect(generator.getTodo).toHaveBeenCalled(); expect(writeFileCalled).toBe(true); expect(result.success).toBe(true); expect(result.tokenCount).toBe(21); }); it('should handle errors during markdown creation', async () => { // Spy on generateMarkdown to reject const generateMarkdownSpy = spyOn(markdownGenerator, 'generateMarkdown').mockRejectedValue(new Error('Generation failed')); const result = await markdownGenerator.createMarkdownDocument(); expect(result.success).toBe(false); expect(result.error).toEqual(new Error('Generation failed')); // Restore the original implementation generateMarkdownSpy.mockRestore(); }); }); });