geoffsee/code-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generated test suite

Browse Source
This commit is contained in:
Geoff Seemueller
2024-11-21 13:36:57 -05:00
parent 4a0c75f3a2
commit 137976a17c
3 changed files with 465 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/TokenCleaner.ts
View File

@@ -7,13 +7,13 @@ export class TokenCleaner {
replacement: string
}[] = []) {
this.patterns = [
{ regex: /\/\/.*$/gm, replacement: '' },
{ regex: /\/\*[\s\S]*?\*\//gm, replacement: '' },
{ regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' },
{ regex: /^\s*[\r\n]/gm, replacement: '' },
{ regex: / +$/gm, replacement: '' },
{ regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' },
{ regex: /^\s*\n+/gm, replacement: '\n' },
{ regex: /\/\/.*$/gm, replacement: '' }, // Single-line comments
{ regex: /\/\*[\s\S]*?\*\//g, replacement: '' }, // Multi-line comments
{ regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, // Console statements
{ regex: /^\s*[\r\n]/gm, replacement: '' }, // Empty lines
{ regex: / +$/gm, replacement: '' }, // Trailing spaces
{ regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, // Import statements
{ regex: /^\s*\n+/gm, replacement: '\n' }, // Multiple newlines
...customPatterns,
];
// eslint-no-no-useless-escape