geoffsee/code-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all tests passing

Browse Source
This commit is contained in:
Geoff Seemueller
2025-01-09 16:09:58 -05:00
parent 994f7d4a8d
commit 5d39f13c98
3 changed files with 114 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/MarkdownGenerator.ts
View File

@@ -175,7 +175,7 @@ export class MarkdownGenerator {
const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length;
console.log(`${filePath}: Tokens[${tokenCount}]`);
}
return cleanedAndRedactedContent;
return cleanedAndRedactedContent.trimEnd();
} catch (error) {
if (this.verbose) {
console.error(`Error reading file ${filePath}:`, error);
@@ -284,4 +284,4 @@ export class MarkdownGenerator {
throw error;
}
}
}
}

74
src/TokenCleaner.ts
View File

@@ -17,33 +17,38 @@ export class TokenCleaner {
...customPatterns,
];
// eslint-no-no-useless-escape
this.secretPatterns = [
{
regex:
/(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{
regex:
/(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{ regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' },
{
regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
replacement: '[REDACTED]',
},
{
regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g,
replacement: '[REDACTED_JWT]',
},
{ regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' },
{
regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g,
replacement: '[REDACTED_BASE64]',
},
...customSecretPatterns,
];
(this.secretPatterns = [
{
regex: /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{
regex: /(?<=const\s+\w+\s*=\s*['"])(eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*)(?=['"])/g,
replacement: '[REDACTED_JWT]',
},
{
regex: /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{
regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
replacement: '[REDACTED]'
},
{
regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
replacement: '[REDACTED]',
},
{
regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi,
replacement: '[REDACTED_HASH]',
},
{
regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g,
replacement: '[REDACTED_BASE64]',
},
...customSecretPatterns,
]);
}
clean(code: string): string {
@@ -61,7 +66,16 @@ export class TokenCleaner {
}
cleanAndRedact(code: string): string {
const cleanedCode = this.clean(code);
return this.redactSecrets(cleanedCode);
// First redact secrets
const redactedCode = this.redactSecrets(code);
// Add pattern to remove lines that only contain redacted content
const redactedLines = /^.*\[REDACTED(?:_[A-Z]+)?\].*$/gm;
const withoutRedactedLines = redactedCode.replace(redactedLines, '');
// Then clean the code
const cleanedCode = this.clean(withoutRedactedLines);
return cleanedCode.trim();
}
}
}