diff --git a/README.md b/README.md index 2e8fe97..3157c65 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,11 @@ > Created to push creative limits. -Process git repository files into markdown with token counting and sensitive -data redaction. +Process git repository files into markdown with token counting and sensitive data redaction. ## Overview -`code-tokenizer-md` is a Node.js tool that processes git repository files, -cleans code, redacts sensitive information, and generates markdown documentation -with token counts. +`code-tokenizer-md` is a TypeScript/Bun tool that processes git repository files, cleans code, redacts sensitive information, and generates markdown documentation with token counts. ```mermaid graph TD @@ -52,26 +49,31 @@ graph TD ## Requirements - Node.js (>=14.0.0) +- Bun runtime - Git repository -- npm or npx ## Installation -```shell -npm install -g code-tokenizer-md -``` + ## Usage -### Quick Start +### CLI ```shell npx code-tokenizer-md ``` +### Library +### +```shell +npm install code-tokenizer-md +``` + + ### Programmatic Usage -```javascript +```typescript import { MarkdownGenerator } from 'code-tokenizer-md'; const generator = new MarkdownGenerator({ @@ -86,10 +88,12 @@ const result = await generator.createMarkdownDocument(); ``` src/ -├── index.js # Main exports -├── TokenCleaner.js # Code cleaning and redaction -├── MarkdownGenerator.js # Markdown generation logic -└── cli.js # CLI implementation +├── index.ts # Main exports +├── TokenCleaner.ts # Code cleaning and redaction +├── MarkdownGenerator.ts # Markdown generation logic +├── cli.ts # CLI implementation +├── fileExclusions.ts # File exclusion patterns +└── fileTypeExclusions.ts # File type exclusions ``` ## Dependencies @@ -97,25 +101,81 @@ src/ ```json { "dependencies": { - "llama3-tokenizer-js": "^1.0.0" + "llama3-tokenizer-js": "^1.0.0", + "micromatch": "^4.0.8" }, "peerDependencies": { "node": ">=14.0.0" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "eslint": "^9.14.0", + "globals": "^15.12.0", + "prettier": "^3.3.3", + "bun": "latest", + "@types/bun": "latest", + "@types/node": "^22.9.1", + "@types/micromatch": "^4.0.9" } } ``` +## Development + +This project uses [bun](https://github.com/oven-sh/bun) for it's toolchain. You should be able to use whatever you want as a consumer of the library. + +### Building +```shell +npm run build +``` + +### Testing + +```shell +npm test +``` + +### Linting and Formatting + +```shell +# Lint +npm run lint + +# Fix linting issues +npm run lint:fix + +# Format code +npm run format + +# Fix all (format + lint) +npm run fix +``` + ## Extending ### Adding Custom Patterns -```javascript +```typescript const generator = new MarkdownGenerator({ customPatterns: [{ regex: /TODO:/g, replacement: '' }], customSecretPatterns: [{ regex: /mySecret/g, replacement: '[REDACTED]' }], }); ``` +### Configuration Options + +```typescript +interface MarkdownGeneratorOptions { + dir?: string; // Project directory + outputFilePath?: string; // Output markdown file path + fileTypeExclusions?: Set; // File types to exclude + fileExclusions?: string[]; // File patterns to exclude + customPatterns?: Record; // Custom cleaning patterns + customSecretPatterns?: Record; // Custom redaction patterns + verbose?: boolean; // Enable verbose logging +} +``` + ## Contributing 1. Fork the repository @@ -126,10 +186,10 @@ const generator = new MarkdownGenerator({ ### Contribution Guidelines -- Follow Node.js best practices +- Write TypeScript code following the project's style - Include appropriate error handling - Add documentation for new features -- Include tests for new functionality (this project needs a suite) +- Include tests for new functionality - Update the README for significant changes ## License @@ -138,4 +198,4 @@ MIT © 2024 Geoff Seemueller ## Note -This tool requires a git repository to function properly. +This tool requires a git repository to function properly. \ No newline at end of file diff --git a/build.ts b/build.ts index 9adf48e..2333267 100644 --- a/build.ts +++ b/build.ts @@ -1,5 +1,12 @@ +import isolatedDecl from 'bun-plugin-isolated-decl'; + await Bun.build({ - entrypoints: ['./src/cli.ts'], + entrypoints: ['./src/cli.ts', './src/index.ts'], outdir: './dist', target: 'node', -}); \ No newline at end of file + plugins: [ + isolatedDecl({ + forceGenerate: true, // Generate declaration files even if there are errors + }) + ], +}); diff --git a/bun.lockb b/bun.lockb index b145f6c..3147607 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 42e53e9..ec99b65 100644 --- a/package.json +++ b/package.json @@ -1,20 +1,30 @@ { "name": "code-tokenizer-md", - "version": "1.0.11", + "version": "1.0.12", "type": "module", - "main": "dist/index.js", + "main": "src/index.ts", + "module": "src/index.ts", + "exports": { + ".": { + "import": { + "types": "./src/index.ts", + "default": "./dist/index.js" + } + } + }, "bin": { "code-tokenizer-md": "./dist/cli.js" }, "files": [ "dist" ], + "types": "", "scripts": { - "build": "rm -rf dist && bun run build.ts", + "build": "rm -rf dist && bun ./build.ts", "test": "bun test", "prepublishOnly": "npm run build", - "dev": "npx .", - "deploy:dev": "pnpm build && pnpm publish .", + "dev": "bun run .", + "deploy:dev": "bun run build && pnpm publish .", "lint": "eslint src/", "lint:fix": "eslint src/ --fix", "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md,yml,yaml}\"", @@ -29,12 +39,15 @@ }, "devDependencies": { "@eslint/js": "^9.14.0", + "@types/bun": "latest", + "@types/micromatch": "^4.0.9", + "@types/node": "^22.9.1", + "bun": "latest", + "bun-plugin-isolated-decl": "^0.1.6", "eslint": "^9.14.0", "globals": "^15.12.0", + "oxc-transform": "^0.37.0", "prettier": "^3.3.3", - "bun": "latest", - "@types/bun": "latest", - "@types/node": "^22.9.1", - "@types/micromatch": "^4.0.9" + "typescript": "^5.6.3" } } diff --git a/src/index.ts b/src/index.ts index 3675345..54b0d84 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,2 +1,2 @@ export { TokenCleaner } from './TokenCleaner'; -export { MarkdownGenerator } from './MarkdownGenerator'; +export { MarkdownGenerator } from './MarkdownGenerator'; \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 238655f..9acf524 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,7 +7,6 @@ "moduleDetection": "force", "jsx": "react-jsx", "allowJs": true, - // Bundler mode "moduleResolution": "bundler", "allowImportingTsExtensions": true,