add eslint

This commit is contained in:
2024-11-07 11:37:56 -05:00
parent 2f809fb82e
commit e747f99984
9 changed files with 906 additions and 141 deletions

3
.prettierignore Normal file
View File

@@ -0,0 +1,3 @@
# Ignore artifacts:
build
coverage

32
.prettierrc Normal file
View File

@@ -0,0 +1,32 @@
{
"semi": true,
"singleQuote": true,
"trailingComma": "es5",
"tabWidth": 2,
"printWidth": 100,
"bracketSpacing": true,
"arrowParens": "avoid",
"endOfLine": "lf",
"jsxSingleQuote": false,
"quoteProps": "as-needed",
"bracketSameLine": false,
"proseWrap": "preserve",
"htmlWhitespaceSensitivity": "css",
"embeddedLanguageFormatting": "auto",
"singleAttributePerLine": false,
"overrides": [
{
"files": "*.md",
"options": {
"proseWrap": "always",
"printWidth": 80
}
},
{
"files": ["*.yml", "*.yaml"],
"options": {
"singleQuote": false
}
}
]
}

71
eslint.config.js Normal file
View File

@@ -0,0 +1,71 @@
import globals from 'globals';
import js from '@eslint/js';
/** @type {import('eslint').Linter.Config[]} */
export default [
{
ignores: ['dist/**', 'node_modules/**', '*.min.js', '*.d.ts'],
},
{
languageOptions: {
ecmaVersion: 2024,
sourceType: 'module',
globals: {
...globals.node,
...globals.browser,
},
},
linterOptions: {
reportUnusedDisableDirectives: true,
},
rules: {
...js.configs.recommended.rules,
'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off',
'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off',
'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
'no-constant-condition': ['error', { checkLoops: false }],
'no-multiple-empty-lines': ['error', { max: 1, maxEOF: 0 }],
'quotes': ['error', 'single', { avoidEscape: true }],
'semi': ['error', 'always'],
'indent': ['error', 2, { SwitchCase: 1 }],
'comma-dangle': ['error', 'always-multiline'],
'arrow-parens': ['error', 'as-needed'], // Changed from 'avoid' to 'as-needed'
'object-curly-spacing': ['error', 'always'],
'array-bracket-spacing': ['error', 'never'],
'space-before-function-paren': ['error', {
anonymous: 'never',
named: 'never',
asyncArrow: 'always',
}],
'no-trailing-spaces': 'error',
'eol-last': ['error', 'always'],
'prefer-const': 'error',
'no-var': 'error',
'eqeqeq': ['error', 'always'],
'curly': ['error', 'all'],
'brace-style': ['error', '1tbs', { allowSingleLine: false }],
'keyword-spacing': ['error', { before: true, after: true }],
'space-infix-ops': 'error',
'comma-spacing': ['error', { before: false, after: true }],
'no-multi-spaces': 'error',
'no-irregular-whitespace': 'error',
'no-mixed-spaces-and-tabs': 'error',
'no-else-return': 'error',
},
},
{
files: ['src/TokenCleaner.js'],
rules: {
'no-useless-escape': 'off'
}
},
{
files: ['**/*.test.js', '**/*.spec.js'],
languageOptions: {
globals: {
...globals.jest,
},
},
},
];

View File

@@ -14,12 +14,22 @@
"test": "echo \"No tests specified\" && exit 0", "test": "echo \"No tests specified\" && exit 0",
"prepublishOnly": "npm run build", "prepublishOnly": "npm run build",
"dev": "node ./src/cli.js", "dev": "node ./src/cli.js",
"deploy:dev": "pnpm publish ." "deploy:dev": "pnpm publish .",
"lint": "eslint . --ext .js,.jsx,.ts,.tsx",
"lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
"format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md,yml,yaml}\"",
"fix": "pnpm format && pnpm lint:fix"
}, },
"dependencies": { "dependencies": {
"llama3-tokenizer-js": "^1.0.0" "llama3-tokenizer-js": "^1.0.0"
}, },
"peerDependencies": { "peerDependencies": {
"node": ">=14.0.0" "node": ">=14.0.0"
},
"devDependencies": {
"@eslint/js": "^9.14.0",
"eslint": "^9.14.0",
"globals": "^15.12.0",
"prettier": "^3.3.3"
} }
} }

594
pnpm-lock.yaml generated
View File

@@ -12,12 +12,483 @@ dependencies:
specifier: '>=14.0.0' specifier: '>=14.0.0'
version: 22.11.0 version: 22.11.0
devDependencies:
'@eslint/js':
specifier: ^9.14.0
version: 9.14.0
eslint:
specifier: ^9.14.0
version: 9.14.0
globals:
specifier: ^15.12.0
version: 15.12.0
prettier:
specifier: ^3.3.3
version: 3.3.3
packages: packages:
/@eslint-community/eslint-utils@4.4.1(eslint@9.14.0):
resolution: {integrity: sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
eslint: ^6.0.0 || ^7.0.0 || >=8.0.0
dependencies:
eslint: 9.14.0
eslint-visitor-keys: 3.4.3
dev: true
/@eslint-community/regexpp@4.12.1:
resolution: {integrity: sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==}
engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}
dev: true
/@eslint/config-array@0.18.0:
resolution: {integrity: sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dependencies:
'@eslint/object-schema': 2.1.4
debug: 4.3.7
minimatch: 3.1.2
transitivePeerDependencies:
- supports-color
dev: true
/@eslint/core@0.7.0:
resolution: {integrity: sha512-xp5Jirz5DyPYlPiKat8jaq0EmYvDXKKpzTbxXMpT9eqlRJkRKIz9AGMdlvYjih+im+QlhWrpvVjl8IPC/lHlUw==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dev: true
/@eslint/eslintrc@3.1.0:
resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dependencies:
ajv: 6.12.6
debug: 4.3.7
espree: 10.3.0
globals: 14.0.0
ignore: 5.3.2
import-fresh: 3.3.0
js-yaml: 4.1.0
minimatch: 3.1.2
strip-json-comments: 3.1.1
transitivePeerDependencies:
- supports-color
dev: true
/@eslint/js@9.14.0:
resolution: {integrity: sha512-pFoEtFWCPyDOl+C6Ift+wC7Ro89otjigCf5vcuWqWgqNSQbRrpjSvdeE6ofLz4dHmyxD5f7gIdGT4+p36L6Twg==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dev: true
/@eslint/object-schema@2.1.4:
resolution: {integrity: sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dev: true
/@eslint/plugin-kit@0.2.2:
resolution: {integrity: sha512-CXtq5nR4Su+2I47WPOlWud98Y5Lv8Kyxp2ukhgFx/eW6Blm18VXJO5WuQylPugRo8nbluoi6GvvxBLqHcvqUUw==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dependencies:
levn: 0.4.1
dev: true
/@humanfs/core@0.19.1:
resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==}
engines: {node: '>=18.18.0'}
dev: true
/@humanfs/node@0.16.6:
resolution: {integrity: sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==}
engines: {node: '>=18.18.0'}
dependencies:
'@humanfs/core': 0.19.1
'@humanwhocodes/retry': 0.3.1
dev: true
/@humanwhocodes/module-importer@1.0.1:
resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==}
engines: {node: '>=12.22'}
dev: true
/@humanwhocodes/retry@0.3.1:
resolution: {integrity: sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==}
engines: {node: '>=18.18'}
dev: true
/@humanwhocodes/retry@0.4.1:
resolution: {integrity: sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==}
engines: {node: '>=18.18'}
dev: true
/@types/estree@1.0.6:
resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==}
dev: true
/@types/json-schema@7.0.15:
resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==}
dev: true
/acorn-jsx@5.3.2(acorn@8.14.0):
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
peerDependencies:
acorn: ^6.0.0 || ^7.0.0 || ^8.0.0
dependencies:
acorn: 8.14.0
dev: true
/acorn@8.14.0:
resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==}
engines: {node: '>=0.4.0'}
hasBin: true
dev: true
/ajv@6.12.6:
resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==}
dependencies:
fast-deep-equal: 3.1.3
fast-json-stable-stringify: 2.1.0
json-schema-traverse: 0.4.1
uri-js: 4.4.1
dev: true
/ansi-styles@4.3.0:
resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
engines: {node: '>=8'}
dependencies:
color-convert: 2.0.1
dev: true
/argparse@2.0.1:
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
dev: true
/balanced-match@1.0.2:
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
dev: true
/brace-expansion@1.1.11:
resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==}
dependencies:
balanced-match: 1.0.2
concat-map: 0.0.1
dev: true
/callsites@3.1.0:
resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==}
engines: {node: '>=6'}
dev: true
/chalk@4.1.2:
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
dependencies:
ansi-styles: 4.3.0
supports-color: 7.2.0
dev: true
/color-convert@2.0.1:
resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
engines: {node: '>=7.0.0'}
dependencies:
color-name: 1.1.4
dev: true
/color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
dev: true
/concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
dev: true
/cross-spawn@7.0.5:
resolution: {integrity: sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==}
engines: {node: '>= 8'}
dependencies:
path-key: 3.1.1
shebang-command: 2.0.0
which: 2.0.2
dev: true
/debug@4.3.7:
resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
engines: {node: '>=6.0'}
peerDependencies:
supports-color: '*'
peerDependenciesMeta:
supports-color:
optional: true
dependencies:
ms: 2.1.3
dev: true
/deep-is@0.1.4:
resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==}
dev: true
/escape-string-regexp@4.0.0:
resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
engines: {node: '>=10'}
dev: true
/eslint-scope@8.2.0:
resolution: {integrity: sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dependencies:
esrecurse: 4.3.0
estraverse: 5.3.0
dev: true
/eslint-visitor-keys@3.4.3:
resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
dev: true
/eslint-visitor-keys@4.2.0:
resolution: {integrity: sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dev: true
/eslint@9.14.0:
resolution: {integrity: sha512-c2FHsVBr87lnUtjP4Yhvk4yEhKrQavGafRA/Se1ouse8PfbfC/Qh9Mxa00yWsZRlqeUB9raXip0aiiUZkgnr9g==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
hasBin: true
peerDependencies:
jiti: '*'
peerDependenciesMeta:
jiti:
optional: true
dependencies:
'@eslint-community/eslint-utils': 4.4.1(eslint@9.14.0)
'@eslint-community/regexpp': 4.12.1
'@eslint/config-array': 0.18.0
'@eslint/core': 0.7.0
'@eslint/eslintrc': 3.1.0
'@eslint/js': 9.14.0
'@eslint/plugin-kit': 0.2.2
'@humanfs/node': 0.16.6
'@humanwhocodes/module-importer': 1.0.1
'@humanwhocodes/retry': 0.4.1
'@types/estree': 1.0.6
'@types/json-schema': 7.0.15
ajv: 6.12.6
chalk: 4.1.2
cross-spawn: 7.0.5
debug: 4.3.7
escape-string-regexp: 4.0.0
eslint-scope: 8.2.0
eslint-visitor-keys: 4.2.0
espree: 10.3.0
esquery: 1.6.0
esutils: 2.0.3
fast-deep-equal: 3.1.3
file-entry-cache: 8.0.0
find-up: 5.0.0
glob-parent: 6.0.2
ignore: 5.3.2
imurmurhash: 0.1.4
is-glob: 4.0.3
json-stable-stringify-without-jsonify: 1.0.1
lodash.merge: 4.6.2
minimatch: 3.1.2
natural-compare: 1.4.0
optionator: 0.9.4
text-table: 0.2.0
transitivePeerDependencies:
- supports-color
dev: true
/espree@10.3.0:
resolution: {integrity: sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
dependencies:
acorn: 8.14.0
acorn-jsx: 5.3.2(acorn@8.14.0)
eslint-visitor-keys: 4.2.0
dev: true
/esquery@1.6.0:
resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==}
engines: {node: '>=0.10'}
dependencies:
estraverse: 5.3.0
dev: true
/esrecurse@4.3.0:
resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==}
engines: {node: '>=4.0'}
dependencies:
estraverse: 5.3.0
dev: true
/estraverse@5.3.0:
resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==}
engines: {node: '>=4.0'}
dev: true
/esutils@2.0.3:
resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==}
engines: {node: '>=0.10.0'}
dev: true
/fast-deep-equal@3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
dev: true
/fast-json-stable-stringify@2.1.0:
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
dev: true
/fast-levenshtein@2.0.6:
resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==}
dev: true
/file-entry-cache@8.0.0:
resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==}
engines: {node: '>=16.0.0'}
dependencies:
flat-cache: 4.0.1
dev: true
/find-up@5.0.0:
resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==}
engines: {node: '>=10'}
dependencies:
locate-path: 6.0.0
path-exists: 4.0.0
dev: true
/flat-cache@4.0.1:
resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==}
engines: {node: '>=16'}
dependencies:
flatted: 3.3.1
keyv: 4.5.4
dev: true
/flatted@3.3.1:
resolution: {integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==}
dev: true
/glob-parent@6.0.2:
resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==}
engines: {node: '>=10.13.0'}
dependencies:
is-glob: 4.0.3
dev: true
/globals@14.0.0:
resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==}
engines: {node: '>=18'}
dev: true
/globals@15.12.0:
resolution: {integrity: sha512-1+gLErljJFhbOVyaetcwJiJ4+eLe45S2E7P5UiZ9xGfeq3ATQf5DOv9G7MH3gGbKQLkzmNh2DxfZwLdw+j6oTQ==}
engines: {node: '>=18'}
dev: true
/has-flag@4.0.0:
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
dev: true
/ignore@5.3.2:
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
engines: {node: '>= 4'}
dev: true
/import-fresh@3.3.0:
resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==}
engines: {node: '>=6'}
dependencies:
parent-module: 1.0.1
resolve-from: 4.0.0
dev: true
/imurmurhash@0.1.4:
resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
engines: {node: '>=0.8.19'}
dev: true
/is-extglob@2.1.1:
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
dev: true
/is-glob@4.0.3:
resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
engines: {node: '>=0.10.0'}
dependencies:
is-extglob: 2.1.1
dev: true
/isexe@2.0.0:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
dev: true
/js-yaml@4.1.0:
resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
hasBin: true
dependencies:
argparse: 2.0.1
dev: true
/json-buffer@3.0.1:
resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==}
dev: true
/json-schema-traverse@0.4.1:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
dev: true
/json-stable-stringify-without-jsonify@1.0.1:
resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==}
dev: true
/keyv@4.5.4:
resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
dependencies:
json-buffer: 3.0.1
dev: true
/levn@0.4.1:
resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
engines: {node: '>= 0.8.0'}
dependencies:
prelude-ls: 1.2.1
type-check: 0.4.0
dev: true
/llama3-tokenizer-js@1.2.0: /llama3-tokenizer-js@1.2.0:
resolution: {integrity: sha512-oMgIgK958UlvoEm3Lz/gAj3QAKpnAMb6YqlY0aTYraSK/c+V3TF3P7IWFQJe4yjM60+2/KoK+EWziec6WQ57/g==} resolution: {integrity: sha512-oMgIgK958UlvoEm3Lz/gAj3QAKpnAMb6YqlY0aTYraSK/c+V3TF3P7IWFQJe4yjM60+2/KoK+EWziec6WQ57/g==}
dev: false dev: false
/locate-path@6.0.0:
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
engines: {node: '>=10'}
dependencies:
p-locate: 5.0.0
dev: true
/lodash.merge@4.6.2:
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
dev: true
/minimatch@3.1.2:
resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==}
dependencies:
brace-expansion: 1.1.11
dev: true
/ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
dev: true
/natural-compare@1.4.0:
resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
dev: true
/node-bin-setup@1.1.3: /node-bin-setup@1.1.3:
resolution: {integrity: sha512-opgw9iSCAzT2+6wJOETCpeRYAQxSopqQ2z+N6BXwIMsQQ7Zj5M8MaafQY8JMlolRR6R1UXg2WmhKp0p9lSOivg==} resolution: {integrity: sha512-opgw9iSCAzT2+6wJOETCpeRYAQxSopqQ2z+N6BXwIMsQQ7Zj5M8MaafQY8JMlolRR6R1UXg2WmhKp0p9lSOivg==}
dev: false dev: false
@@ -30,3 +501,126 @@ packages:
dependencies: dependencies:
node-bin-setup: 1.1.3 node-bin-setup: 1.1.3
dev: false dev: false
/optionator@0.9.4:
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
engines: {node: '>= 0.8.0'}
dependencies:
deep-is: 0.1.4
fast-levenshtein: 2.0.6
levn: 0.4.1
prelude-ls: 1.2.1
type-check: 0.4.0
word-wrap: 1.2.5
dev: true
/p-limit@3.1.0:
resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
engines: {node: '>=10'}
dependencies:
yocto-queue: 0.1.0
dev: true
/p-locate@5.0.0:
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
engines: {node: '>=10'}
dependencies:
p-limit: 3.1.0
dev: true
/parent-module@1.0.1:
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
engines: {node: '>=6'}
dependencies:
callsites: 3.1.0
dev: true
/path-exists@4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
dev: true
/path-key@3.1.1:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
dev: true
/prelude-ls@1.2.1:
resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
engines: {node: '>= 0.8.0'}
dev: true
/prettier@3.3.3:
resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==}
engines: {node: '>=14'}
hasBin: true
dev: true
/punycode@2.3.1:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
dev: true
/resolve-from@4.0.0:
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
engines: {node: '>=4'}
dev: true
/shebang-command@2.0.0:
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
engines: {node: '>=8'}
dependencies:
shebang-regex: 3.0.0
dev: true
/shebang-regex@3.0.0:
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
engines: {node: '>=8'}
dev: true
/strip-json-comments@3.1.1:
resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
engines: {node: '>=8'}
dev: true
/supports-color@7.2.0:
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
engines: {node: '>=8'}
dependencies:
has-flag: 4.0.0
dev: true
/text-table@0.2.0:
resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==}
dev: true
/type-check@0.4.0:
resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==}
engines: {node: '>= 0.8.0'}
dependencies:
prelude-ls: 1.2.1
dev: true
/uri-js@4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
dependencies:
punycode: 2.3.1
dev: true
/which@2.0.2:
resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
engines: {node: '>= 8'}
hasBin: true
dependencies:
isexe: 2.0.0
dev: true
/word-wrap@1.2.5:
resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
engines: {node: '>=0.10.0'}
dev: true
/yocto-queue@0.1.0:
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
engines: {node: '>=10'}
dev: true

View File

@@ -9,8 +9,29 @@ export class MarkdownGenerator {
constructor(options = {}) { constructor(options = {}) {
this.dir = options.dir || '.'; this.dir = options.dir || '.';
this.outputFilePath = options.outputFilePath || './prompt.md'; this.outputFilePath = options.outputFilePath || './prompt.md';
this.fileTypeExclusions = new Set(options.fileTypeExclusions || ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp', '.tiff', '.lockb', '.yaml', '.ico', '.ttf', '.css']); this.fileTypeExclusions = new Set(
this.fileExclusions = options.fileExclusions || ['prompt.js', '.gitignore', '.env', '.dev.vars']; options.fileTypeExclusions || [
'.jpg',
'.jpeg',
'.png',
'.gif',
'.bmp',
'.svg',
'.webp',
'.tiff',
'.lockb',
'.yaml',
'.ico',
'.ttf',
'.css',
],
);
this.fileExclusions = options.fileExclusions || [
'prompt.js',
'.gitignore',
'.env',
'.dev.vars',
];
this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns); this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns);
this.verbose = options.verbose ?? true; this.verbose = options.verbose ?? true;
} }
@@ -19,14 +40,18 @@ export class MarkdownGenerator {
try { try {
const output = this.execCommand('git ls-files'); const output = this.execCommand('git ls-files');
const trackedFiles = output.split('\n').filter(file => file.length > 0); const trackedFiles = output.split('\n').filter(file => file.length > 0);
if (this.verbose) console.log(`Total tracked files: ${trackedFiles.length}`); if (this.verbose) {
console.log(`Total tracked files: ${trackedFiles.length}`);
}
return trackedFiles.filter(file => { return trackedFiles.filter(file => {
const fileExt = path.extname(file).toLowerCase(); const fileExt = path.extname(file).toLowerCase();
const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern)); const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern));
return !this.fileTypeExclusions.has(fileExt) && !isExcluded; return !this.fileTypeExclusions.has(fileExt) && !isExcluded;
}); });
} catch (error) { } catch (error) {
if (this.verbose) console.error('Error fetching tracked files:', error); if (this.verbose) {
console.error('Error fetching tracked files:', error);
}
return []; return [];
} }
} }
@@ -53,14 +78,18 @@ export class MarkdownGenerator {
} }
return cleanedAndRedactedContent; return cleanedAndRedactedContent;
} catch (error) { } catch (error) {
if (this.verbose) console.error(`Error reading file ${filePath}:`, error); if (this.verbose) {
console.error(`Error reading file ${filePath}:`, error);
}
return ''; return '';
} }
} }
async generateMarkdown() { async generateMarkdown() {
const trackedFiles = await this.getTrackedFiles(); const trackedFiles = await this.getTrackedFiles();
if (this.verbose) console.log(`Generating markdown for ${trackedFiles.length} files`); if (this.verbose) {
console.log(`Generating markdown for ${trackedFiles.length} files`);
}
let markdownContent = '# Project Files\n\n'; let markdownContent = '# Project Files\n\n';
for (const file of trackedFiles) { for (const file of trackedFiles) {
@@ -72,16 +101,16 @@ export class MarkdownGenerator {
async getTodo() { async getTodo() {
try { try {
console.log("getting project todo") console.log('getting project todo');
return await readFile('./todo', 'utf-8'); return await readFile('./todo', 'utf-8');
} catch (error) { } catch (error) {
if (error.code === 'ENOENT') { // File does not exist if (error.code === 'ENOENT') {
// File does not exist
console.log("File not found, creating a new 'todo' file."); console.log("File not found, creating a new 'todo' file.");
await writeFile('./todo', ''); // Create an empty 'todo' file await writeFile('./todo', ''); // Create an empty 'todo' file
return this.getTodo(); // Call the function again return this.getTodo(); // Call the function again
} else {
console.error(`Error reading todo file:`, error);
} }
console.error('Error reading todo file:', error);
} }
} }
@@ -98,7 +127,9 @@ export class MarkdownGenerator {
} }
return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length }; return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length };
} catch (error) { } catch (error) {
if (this.verbose) console.error('Error writing markdown document:', error); if (this.verbose) {
console.error('Error writing markdown document:', error);
}
return { success: false, error }; return { success: false, error };
} }
} }
@@ -107,7 +138,9 @@ export class MarkdownGenerator {
try { try {
return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim(); return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim();
} catch (error) { } catch (error) {
if (this.verbose) console.error(`Error executing command: ${command}`, error); if (this.verbose) {
console.error(`Error executing command: ${command}`, error);
}
throw error; throw error;
} }
} }

View File

@@ -8,29 +8,50 @@ export class TokenCleaner {
{ regex: / +$/gm, replacement: '' }, { regex: / +$/gm, replacement: '' },
{ regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' },
{ regex: /^\s*\n+/gm, replacement: '\n' }, { regex: /^\s*\n+/gm, replacement: '\n' },
...customPatterns ...customPatterns,
]; ];
// eslint-no-no-useless-escape
this.secretPatterns = [ this.secretPatterns = [
{ regex: /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' }, {
{ regex: /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' }, regex:
/(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{
regex:
/(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{ regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' }, { regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' },
{ regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' }, {
{ regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g, replacement: '[REDACTED_JWT]' }, regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
replacement: '[REDACTED]',
},
{
regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g,
replacement: '[REDACTED_JWT]',
},
{ regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' }, { regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' },
{ regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g, replacement: '[REDACTED_BASE64]' }, {
...customSecretPatterns regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g,
replacement: '[REDACTED_BASE64]',
},
...customSecretPatterns,
]; ];
} }
clean(code) { clean(code) {
return this.patterns.reduce((cleanCode, pattern) => return this.patterns.reduce(
cleanCode.replace(pattern.regex, pattern.replacement), code); (cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement),
code,
);
} }
redactSecrets(code) { redactSecrets(code) {
return this.secretPatterns.reduce((redactedCode, pattern) => return this.secretPatterns.reduce(
redactedCode.replace(pattern.regex, pattern.replacement), code); (redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement),
code,
);
} }
cleanAndRedact(code) { cleanAndRedact(code) {

View File

@@ -2,7 +2,8 @@
import { MarkdownGenerator } from './MarkdownGenerator.js'; import { MarkdownGenerator } from './MarkdownGenerator.js';
const generator = new MarkdownGenerator(); const generator = new MarkdownGenerator();
generator.createMarkdownDocument() generator
.createMarkdownDocument()
.then(result => { .then(result => {
if (!result.success) { if (!result.success) {
process.exit(1); process.exit(1);