From e747f99984db7043b86622024ba0c1f9890f22df Mon Sep 17 00:00:00 2001 From: Geoff Seemueller Date: Thu, 7 Nov 2024 11:37:56 -0500 Subject: [PATCH] add eslint --- .prettierignore | 3 + .prettierrc | 32 +++ eslint.config.js | 71 +++++ package.json | 12 +- pnpm-lock.yaml | 594 +++++++++++++++++++++++++++++++++++++++ src/MarkdownGenerator.js | 219 +++++++++------ src/TokenCleaner.js | 93 +++--- src/cli.js | 21 +- src/index.js | 2 +- 9 files changed, 906 insertions(+), 141 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 eslint.config.js diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..1b8ac88 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +# Ignore artifacts: +build +coverage diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..10411d7 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,32 @@ +{ + "semi": true, + "singleQuote": true, + "trailingComma": "es5", + "tabWidth": 2, + "printWidth": 100, + "bracketSpacing": true, + "arrowParens": "avoid", + "endOfLine": "lf", + "jsxSingleQuote": false, + "quoteProps": "as-needed", + "bracketSameLine": false, + "proseWrap": "preserve", + "htmlWhitespaceSensitivity": "css", + "embeddedLanguageFormatting": "auto", + "singleAttributePerLine": false, + "overrides": [ + { + "files": "*.md", + "options": { + "proseWrap": "always", + "printWidth": 80 + } + }, + { + "files": ["*.yml", "*.yaml"], + "options": { + "singleQuote": false + } + } + ] +} \ No newline at end of file diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..dcf8825 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,71 @@ +import globals from 'globals'; +import js from '@eslint/js'; + +/** @type {import('eslint').Linter.Config[]} */ +export default [ + { + ignores: ['dist/**', 'node_modules/**', '*.min.js', '*.d.ts'], + }, + { + languageOptions: { + ecmaVersion: 2024, + sourceType: 'module', + globals: { + ...globals.node, + ...globals.browser, + }, + }, + linterOptions: { + reportUnusedDisableDirectives: true, + }, + rules: { + ...js.configs.recommended.rules, + 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', + 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', + 'no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + 'no-constant-condition': ['error', { checkLoops: false }], + 'no-multiple-empty-lines': ['error', { max: 1, maxEOF: 0 }], + 'quotes': ['error', 'single', { avoidEscape: true }], + 'semi': ['error', 'always'], + 'indent': ['error', 2, { SwitchCase: 1 }], + 'comma-dangle': ['error', 'always-multiline'], + 'arrow-parens': ['error', 'as-needed'], // Changed from 'avoid' to 'as-needed' + 'object-curly-spacing': ['error', 'always'], + 'array-bracket-spacing': ['error', 'never'], + 'space-before-function-paren': ['error', { + anonymous: 'never', + named: 'never', + asyncArrow: 'always', + }], + 'no-trailing-spaces': 'error', + 'eol-last': ['error', 'always'], + 'prefer-const': 'error', + 'no-var': 'error', + 'eqeqeq': ['error', 'always'], + 'curly': ['error', 'all'], + 'brace-style': ['error', '1tbs', { allowSingleLine: false }], + 'keyword-spacing': ['error', { before: true, after: true }], + 'space-infix-ops': 'error', + 'comma-spacing': ['error', { before: false, after: true }], + 'no-multi-spaces': 'error', + 'no-irregular-whitespace': 'error', + 'no-mixed-spaces-and-tabs': 'error', + 'no-else-return': 'error', + }, + }, + { + files: ['src/TokenCleaner.js'], + rules: { + 'no-useless-escape': 'off' + } + }, + { + files: ['**/*.test.js', '**/*.spec.js'], + + languageOptions: { + globals: { + ...globals.jest, + }, + }, + }, +]; \ No newline at end of file diff --git a/package.json b/package.json index 6607b52..7204543 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,22 @@ "test": "echo \"No tests specified\" && exit 0", "prepublishOnly": "npm run build", "dev": "node ./src/cli.js", - "deploy:dev": "pnpm publish ." + "deploy:dev": "pnpm publish .", + "lint": "eslint . --ext .js,.jsx,.ts,.tsx", + "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix", + "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md,yml,yaml}\"", + "fix": "pnpm format && pnpm lint:fix" }, "dependencies": { "llama3-tokenizer-js": "^1.0.0" }, "peerDependencies": { "node": ">=14.0.0" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "eslint": "^9.14.0", + "globals": "^15.12.0", + "prettier": "^3.3.3" } } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43a16c8..c2c693f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,12 +12,483 @@ dependencies: specifier: '>=14.0.0' version: 22.11.0 +devDependencies: + '@eslint/js': + specifier: ^9.14.0 + version: 9.14.0 + eslint: + specifier: ^9.14.0 + version: 9.14.0 + globals: + specifier: ^15.12.0 + version: 15.12.0 + prettier: + specifier: ^3.3.3 + version: 3.3.3 + packages: + /@eslint-community/eslint-utils@4.4.1(eslint@9.14.0): + resolution: {integrity: sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + dependencies: + eslint: 9.14.0 + eslint-visitor-keys: 3.4.3 + dev: true + + /@eslint-community/regexpp@4.12.1: + resolution: {integrity: sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + dev: true + + /@eslint/config-array@0.18.0: + resolution: {integrity: sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dependencies: + '@eslint/object-schema': 2.1.4 + debug: 4.3.7 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + dev: true + + /@eslint/core@0.7.0: + resolution: {integrity: sha512-xp5Jirz5DyPYlPiKat8jaq0EmYvDXKKpzTbxXMpT9eqlRJkRKIz9AGMdlvYjih+im+QlhWrpvVjl8IPC/lHlUw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dev: true + + /@eslint/eslintrc@3.1.0: + resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dependencies: + ajv: 6.12.6 + debug: 4.3.7 + espree: 10.3.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.0 + js-yaml: 4.1.0 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + dev: true + + /@eslint/js@9.14.0: + resolution: {integrity: sha512-pFoEtFWCPyDOl+C6Ift+wC7Ro89otjigCf5vcuWqWgqNSQbRrpjSvdeE6ofLz4dHmyxD5f7gIdGT4+p36L6Twg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dev: true + + /@eslint/object-schema@2.1.4: + resolution: {integrity: sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dev: true + + /@eslint/plugin-kit@0.2.2: + resolution: {integrity: sha512-CXtq5nR4Su+2I47WPOlWud98Y5Lv8Kyxp2ukhgFx/eW6Blm18VXJO5WuQylPugRo8nbluoi6GvvxBLqHcvqUUw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dependencies: + levn: 0.4.1 + dev: true + + /@humanfs/core@0.19.1: + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + dev: true + + /@humanfs/node@0.16.6: + resolution: {integrity: sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==} + engines: {node: '>=18.18.0'} + dependencies: + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.3.1 + dev: true + + /@humanwhocodes/module-importer@1.0.1: + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + dev: true + + /@humanwhocodes/retry@0.3.1: + resolution: {integrity: sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==} + engines: {node: '>=18.18'} + dev: true + + /@humanwhocodes/retry@0.4.1: + resolution: {integrity: sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==} + engines: {node: '>=18.18'} + dev: true + + /@types/estree@1.0.6: + resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} + dev: true + + /@types/json-schema@7.0.15: + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + dev: true + + /acorn-jsx@5.3.2(acorn@8.14.0): + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + dependencies: + acorn: 8.14.0 + dev: true + + /acorn@8.14.0: + resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: true + + /ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + dev: true + + /ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + dependencies: + color-convert: 2.0.1 + dev: true + + /argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: true + + /balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + dev: true + + /brace-expansion@1.1.11: + resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + dev: true + + /callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + dev: true + + /chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + dev: true + + /color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + dependencies: + color-name: 1.1.4 + dev: true + + /color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: true + + /concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + dev: true + + /cross-spawn@7.0.5: + resolution: {integrity: sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==} + engines: {node: '>= 8'} + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + dev: true + + /debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + dev: true + + /deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + dev: true + + /escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + dev: true + + /eslint-scope@8.2.0: + resolution: {integrity: sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + dev: true + + /eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dev: true + + /eslint-visitor-keys@4.2.0: + resolution: {integrity: sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dev: true + + /eslint@9.14.0: + resolution: {integrity: sha512-c2FHsVBr87lnUtjP4Yhvk4yEhKrQavGafRA/Se1ouse8PfbfC/Qh9Mxa00yWsZRlqeUB9raXip0aiiUZkgnr9g==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + dependencies: + '@eslint-community/eslint-utils': 4.4.1(eslint@9.14.0) + '@eslint-community/regexpp': 4.12.1 + '@eslint/config-array': 0.18.0 + '@eslint/core': 0.7.0 + '@eslint/eslintrc': 3.1.0 + '@eslint/js': 9.14.0 + '@eslint/plugin-kit': 0.2.2 + '@humanfs/node': 0.16.6 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.1 + '@types/estree': 1.0.6 + '@types/json-schema': 7.0.15 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.5 + debug: 4.3.7 + escape-string-regexp: 4.0.0 + eslint-scope: 8.2.0 + eslint-visitor-keys: 4.2.0 + espree: 10.3.0 + esquery: 1.6.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.4 + text-table: 0.2.0 + transitivePeerDependencies: + - supports-color + dev: true + + /espree@10.3.0: + resolution: {integrity: sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + dependencies: + acorn: 8.14.0 + acorn-jsx: 5.3.2(acorn@8.14.0) + eslint-visitor-keys: 4.2.0 + dev: true + + /esquery@1.6.0: + resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==} + engines: {node: '>=0.10'} + dependencies: + estraverse: 5.3.0 + dev: true + + /esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + dependencies: + estraverse: 5.3.0 + dev: true + + /estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + dev: true + + /esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + dev: true + + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: true + + /fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + dev: true + + /fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + dev: true + + /file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + dependencies: + flat-cache: 4.0.1 + dev: true + + /find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + dev: true + + /flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + dependencies: + flatted: 3.3.1 + keyv: 4.5.4 + dev: true + + /flatted@3.3.1: + resolution: {integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==} + dev: true + + /glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + dependencies: + is-glob: 4.0.3 + dev: true + + /globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + dev: true + + /globals@15.12.0: + resolution: {integrity: sha512-1+gLErljJFhbOVyaetcwJiJ4+eLe45S2E7P5UiZ9xGfeq3ATQf5DOv9G7MH3gGbKQLkzmNh2DxfZwLdw+j6oTQ==} + engines: {node: '>=18'} + dev: true + + /has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + dev: true + + /ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + dev: true + + /import-fresh@3.3.0: + resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} + engines: {node: '>=6'} + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + dev: true + + /imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + dev: true + + /is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + dev: true + + /is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + dependencies: + is-extglob: 2.1.1 + dev: true + + /isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + dev: true + + /js-yaml@4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: true + + /json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + dev: true + + /json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + dev: true + + /json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + dev: true + + /keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + dependencies: + json-buffer: 3.0.1 + dev: true + + /levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + dev: true + /llama3-tokenizer-js@1.2.0: resolution: {integrity: sha512-oMgIgK958UlvoEm3Lz/gAj3QAKpnAMb6YqlY0aTYraSK/c+V3TF3P7IWFQJe4yjM60+2/KoK+EWziec6WQ57/g==} dev: false + /locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + dependencies: + p-locate: 5.0.0 + dev: true + + /lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + dev: true + + /minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + dependencies: + brace-expansion: 1.1.11 + dev: true + + /ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + dev: true + + /natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + dev: true + /node-bin-setup@1.1.3: resolution: {integrity: sha512-opgw9iSCAzT2+6wJOETCpeRYAQxSopqQ2z+N6BXwIMsQQ7Zj5M8MaafQY8JMlolRR6R1UXg2WmhKp0p9lSOivg==} dev: false @@ -30,3 +501,126 @@ packages: dependencies: node-bin-setup: 1.1.3 dev: false + + /optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + dev: true + + /p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + dependencies: + yocto-queue: 0.1.0 + dev: true + + /p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + dependencies: + p-limit: 3.1.0 + dev: true + + /parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + dependencies: + callsites: 3.1.0 + dev: true + + /path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + dev: true + + /path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + dev: true + + /prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + dev: true + + /prettier@3.3.3: + resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} + engines: {node: '>=14'} + hasBin: true + dev: true + + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + dev: true + + /resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + dev: true + + /shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + dependencies: + shebang-regex: 3.0.0 + dev: true + + /shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + dev: true + + /strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + dev: true + + /supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + dependencies: + has-flag: 4.0.0 + dev: true + + /text-table@0.2.0: + resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} + dev: true + + /type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + dependencies: + prelude-ls: 1.2.1 + dev: true + + /uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + dependencies: + punycode: 2.3.1 + dev: true + + /which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: true + + /word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + dev: true + + /yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + dev: true diff --git a/src/MarkdownGenerator.js b/src/MarkdownGenerator.js index 4af571b..d8de623 100644 --- a/src/MarkdownGenerator.js +++ b/src/MarkdownGenerator.js @@ -6,109 +6,142 @@ import llama3Tokenizer from 'llama3-tokenizer-js'; import { TokenCleaner } from './TokenCleaner.js'; export class MarkdownGenerator { - constructor(options = {}) { - this.dir = options.dir || '.'; - this.outputFilePath = options.outputFilePath || './prompt.md'; - this.fileTypeExclusions = new Set(options.fileTypeExclusions || ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp', '.tiff', '.lockb', '.yaml', '.ico', '.ttf', '.css']); - this.fileExclusions = options.fileExclusions || ['prompt.js', '.gitignore', '.env', '.dev.vars']; - this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns); - this.verbose = options.verbose ?? true; - } + constructor(options = {}) { + this.dir = options.dir || '.'; + this.outputFilePath = options.outputFilePath || './prompt.md'; + this.fileTypeExclusions = new Set( + options.fileTypeExclusions || [ + '.jpg', + '.jpeg', + '.png', + '.gif', + '.bmp', + '.svg', + '.webp', + '.tiff', + '.lockb', + '.yaml', + '.ico', + '.ttf', + '.css', + ], + ); + this.fileExclusions = options.fileExclusions || [ + 'prompt.js', + '.gitignore', + '.env', + '.dev.vars', + ]; + this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns); + this.verbose = options.verbose ?? true; + } - async getTrackedFiles() { - try { - const output = this.execCommand('git ls-files'); - const trackedFiles = output.split('\n').filter(file => file.length > 0); - if (this.verbose) console.log(`Total tracked files: ${trackedFiles.length}`); - return trackedFiles.filter(file => { - const fileExt = path.extname(file).toLowerCase(); - const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern)); - return !this.fileTypeExclusions.has(fileExt) && !isExcluded; - }); - } catch (error) { - if (this.verbose) console.error('Error fetching tracked files:', error); - return []; - } + async getTrackedFiles() { + try { + const output = this.execCommand('git ls-files'); + const trackedFiles = output.split('\n').filter(file => file.length > 0); + if (this.verbose) { + console.log(`Total tracked files: ${trackedFiles.length}`); + } + return trackedFiles.filter(file => { + const fileExt = path.extname(file).toLowerCase(); + const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern)); + return !this.fileTypeExclusions.has(fileExt) && !isExcluded; + }); + } catch (error) { + if (this.verbose) { + console.error('Error fetching tracked files:', error); + } + return []; } + } - isFileExcluded(filePath, pattern) { - if (pattern.endsWith('/*')) { - const directory = pattern.slice(0, -2); - return filePath.startsWith(directory); - } - if (pattern.includes('/*')) { - const [directory, ext] = pattern.split('/*'); - return filePath.startsWith(directory) && filePath.endsWith(ext); - } - return filePath === pattern; + isFileExcluded(filePath, pattern) { + if (pattern.endsWith('/*')) { + const directory = pattern.slice(0, -2); + return filePath.startsWith(directory); } - - async readFileContent(filePath) { - try { - const content = await fs.readFile(filePath, 'utf-8'); - const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content); - if (this.verbose) { - const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length; - console.log(`${filePath}: Tokens[${tokenCount}]`); - } - return cleanedAndRedactedContent; - } catch (error) { - if (this.verbose) console.error(`Error reading file ${filePath}:`, error); - return ''; - } + if (pattern.includes('/*')) { + const [directory, ext] = pattern.split('/*'); + return filePath.startsWith(directory) && filePath.endsWith(ext); } + return filePath === pattern; + } - async generateMarkdown() { - const trackedFiles = await this.getTrackedFiles(); - if (this.verbose) console.log(`Generating markdown for ${trackedFiles.length} files`); - let markdownContent = '# Project Files\n\n'; - - for (const file of trackedFiles) { - const content = await this.readFileContent(path.join(this.dir, file)); - markdownContent += `## ${file}\n~~~\n${content.trim()}\n~~~\n`; - } - return markdownContent; + async readFileContent(filePath) { + try { + const content = await fs.readFile(filePath, 'utf-8'); + const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content); + if (this.verbose) { + const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length; + console.log(`${filePath}: Tokens[${tokenCount}]`); + } + return cleanedAndRedactedContent; + } catch (error) { + if (this.verbose) { + console.error(`Error reading file ${filePath}:`, error); + } + return ''; } + } - async getTodo() { - try { - console.log("getting project todo") - return await readFile('./todo', 'utf-8'); - } catch (error) { - if (error.code === 'ENOENT') { // File does not exist - console.log("File not found, creating a new 'todo' file."); - await writeFile('./todo', ''); // Create an empty 'todo' file - return this.getTodo(); // Call the function again - } else { - console.error(`Error reading todo file:`, error); - } - } + async generateMarkdown() { + const trackedFiles = await this.getTrackedFiles(); + if (this.verbose) { + console.log(`Generating markdown for ${trackedFiles.length} files`); } + let markdownContent = '# Project Files\n\n'; - async createMarkdownDocument() { - try { - const codeMarkdown = await this.generateMarkdown(); - const todos = await this.getTodo(); - const markdown = codeMarkdown + `\n---\n${todos}\n`; - await fs.writeFile(this.outputFilePath, markdown); - if (this.verbose) { - console.log(`Markdown document created at ${this.outputFilePath}`); - const totalTokens = llama3Tokenizer.encode(markdown).length; - console.log({total_tokens: totalTokens}); - } - return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length }; - } catch (error) { - if (this.verbose) console.error('Error writing markdown document:', error); - return { success: false, error }; - } + for (const file of trackedFiles) { + const content = await this.readFileContent(path.join(this.dir, file)); + markdownContent += `## ${file}\n~~~\n${content.trim()}\n~~~\n`; } + return markdownContent; + } - execCommand(command) { - try { - return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim(); - } catch (error) { - if (this.verbose) console.error(`Error executing command: ${command}`, error); - throw error; - } + async getTodo() { + try { + console.log('getting project todo'); + return await readFile('./todo', 'utf-8'); + } catch (error) { + if (error.code === 'ENOENT') { + // File does not exist + console.log("File not found, creating a new 'todo' file."); + await writeFile('./todo', ''); // Create an empty 'todo' file + return this.getTodo(); // Call the function again + } + console.error('Error reading todo file:', error); } -} \ No newline at end of file + } + + async createMarkdownDocument() { + try { + const codeMarkdown = await this.generateMarkdown(); + const todos = await this.getTodo(); + const markdown = codeMarkdown + `\n---\n${todos}\n`; + await fs.writeFile(this.outputFilePath, markdown); + if (this.verbose) { + console.log(`Markdown document created at ${this.outputFilePath}`); + const totalTokens = llama3Tokenizer.encode(markdown).length; + console.log({ total_tokens: totalTokens }); + } + return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length }; + } catch (error) { + if (this.verbose) { + console.error('Error writing markdown document:', error); + } + return { success: false, error }; + } + } + + execCommand(command) { + try { + return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim(); + } catch (error) { + if (this.verbose) { + console.error(`Error executing command: ${command}`, error); + } + throw error; + } + } +} diff --git a/src/TokenCleaner.js b/src/TokenCleaner.js index 5273bc2..88a6205 100644 --- a/src/TokenCleaner.js +++ b/src/TokenCleaner.js @@ -1,40 +1,61 @@ export class TokenCleaner { - constructor(customPatterns = [], customSecretPatterns = []) { - this.patterns = [ - { regex: /\/\/.*$/gm, replacement: '' }, - { regex: /\/\*[\s\S]*?\*\//gm, replacement: '' }, - { regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, - { regex: /^\s*[\r\n]/gm, replacement: '' }, - { regex: / +$/gm, replacement: '' }, - { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, - { regex: /^\s*\n+/gm, replacement: '\n' }, - ...customPatterns - ]; + constructor(customPatterns = [], customSecretPatterns = []) { + this.patterns = [ + { regex: /\/\/.*$/gm, replacement: '' }, + { regex: /\/\*[\s\S]*?\*\//gm, replacement: '' }, + { regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, + { regex: /^\s*[\r\n]/gm, replacement: '' }, + { regex: / +$/gm, replacement: '' }, + { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, + { regex: /^\s*\n+/gm, replacement: '\n' }, + ...customPatterns, + ]; + // eslint-no-no-useless-escape + this.secretPatterns = [ + { + regex: + /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi, + replacement: '[REDACTED]', + }, + { + regex: + /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi, + replacement: '[REDACTED]', + }, + { regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' }, + { + regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, + replacement: '[REDACTED]', + }, + { + regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g, + replacement: '[REDACTED_JWT]', + }, + { regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' }, + { + regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g, + replacement: '[REDACTED_BASE64]', + }, + ...customSecretPatterns, + ]; + } - this.secretPatterns = [ - { regex: /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' }, - { regex: /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' }, - { regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' }, - { regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' }, - { regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g, replacement: '[REDACTED_JWT]' }, - { regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' }, - { regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g, replacement: '[REDACTED_BASE64]' }, - ...customSecretPatterns - ]; - } + clean(code) { + return this.patterns.reduce( + (cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement), + code, + ); + } - clean(code) { - return this.patterns.reduce((cleanCode, pattern) => - cleanCode.replace(pattern.regex, pattern.replacement), code); - } + redactSecrets(code) { + return this.secretPatterns.reduce( + (redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement), + code, + ); + } - redactSecrets(code) { - return this.secretPatterns.reduce((redactedCode, pattern) => - redactedCode.replace(pattern.regex, pattern.replacement), code); - } - - cleanAndRedact(code) { - const cleanedCode = this.clean(code); - return this.redactSecrets(cleanedCode); - } -} \ No newline at end of file + cleanAndRedact(code) { + const cleanedCode = this.clean(code); + return this.redactSecrets(cleanedCode); + } +} diff --git a/src/cli.js b/src/cli.js index 00740dd..bbac1e5 100755 --- a/src/cli.js +++ b/src/cli.js @@ -2,13 +2,14 @@ import { MarkdownGenerator } from './MarkdownGenerator.js'; const generator = new MarkdownGenerator(); -generator.createMarkdownDocument() - .then(result => { - if (!result.success) { - process.exit(1); - } - }) - .catch(error => { - console.error('Error:', error); - process.exit(1); - }); +generator + .createMarkdownDocument() + .then(result => { + if (!result.success) { + process.exit(1); + } + }) + .catch(error => { + console.error('Error:', error); + process.exit(1); + }); diff --git a/src/index.js b/src/index.js index 2416cc6..f4237ff 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,2 @@ export { TokenCleaner } from './TokenCleaner.js'; -export { MarkdownGenerator } from './MarkdownGenerator.js'; \ No newline at end of file +export { MarkdownGenerator } from './MarkdownGenerator.js';