This commit is contained in:
2026-06-01 20:47:26 -05:00
parent 108b8243a9
commit eb7de776e4
5 changed files with 201 additions and 49 deletions
+68 -18
View File
@@ -6,12 +6,29 @@ function estimateTokens(text: string): number {
}
const SKIP_EXTENSIONS = new Set([
"lock", "sum",
"lock",
"sum",
"map",
"png", "jpg", "jpeg", "gif", "svg", "ico", "webp",
"woff", "woff2", "ttf", "eot",
"pdf", "zip", "tar", "gz",
"bin", "exe", "dll", "so", "dylib",
"png",
"jpg",
"jpeg",
"gif",
"svg",
"ico",
"webp",
"woff",
"woff2",
"ttf",
"eot",
"pdf",
"zip",
"tar",
"gz",
"bin",
"exe",
"dll",
"so",
"dylib",
]);
const SKIP_PATTERNS = [
@@ -25,35 +42,53 @@ const SKIP_PATTERNS = [
function shouldSkip(filename: string): boolean {
if (!filename.includes(".")) return true; // no extension = likely binary
const ext = filename.split(".").pop()!.toLowerCase();
return SKIP_EXTENSIONS.has(ext) || SKIP_PATTERNS.some((p) => p.test(filename));
return (
SKIP_EXTENSIONS.has(ext) || SKIP_PATTERNS.some((p) => p.test(filename))
);
}
const EXT_TO_LANG: Record<string, string> = {
ts: "TypeScript", tsx: "TypeScript",
js: "JavaScript", jsx: "JavaScript", mjs: "JavaScript", cjs: "JavaScript",
ts: "TypeScript",
tsx: "TypeScript",
js: "JavaScript",
jsx: "JavaScript",
mjs: "JavaScript",
cjs: "JavaScript",
py: "Python",
go: "Go",
rs: "Rust",
rb: "Ruby",
java: "Java",
kt: "Kotlin", kts: "Kotlin",
kt: "Kotlin",
kts: "Kotlin",
swift: "Swift",
cs: "C#",
cpp: "C++", cc: "C++", cxx: "C++", hpp: "C++",
cpp: "C++",
cc: "C++",
cxx: "C++",
hpp: "C++",
c: "C",
php: "PHP",
sh: "Shell", bash: "Shell", zsh: "Shell",
yaml: "YAML", yml: "YAML",
sh: "Shell",
bash: "Shell",
zsh: "Shell",
yaml: "YAML",
yml: "YAML",
json: "JSON",
md: "Markdown",
sql: "SQL",
html: "HTML", htm: "HTML",
css: "CSS", scss: "CSS", sass: "CSS", less: "CSS",
html: "HTML",
htm: "HTML",
css: "CSS",
scss: "CSS",
sass: "CSS",
less: "CSS",
vue: "Vue",
svelte: "Svelte",
toml: "TOML",
xml: "XML",
tf: "Terraform", hcl: "HCL",
tf: "Terraform",
hcl: "HCL",
};
function detectLanguage(filename: string): string {
@@ -68,6 +103,7 @@ export interface DiffFile {
}
export function parseDiff(rawDiff: string): DiffFile[] {
console.log("Parsing diff...");
const files: DiffFile[] = [];
const sections = rawDiff.split(/^diff --git /m).filter((s) => s.trim());
@@ -82,7 +118,10 @@ export function parseDiff(rawDiff: string): DiffFile[] {
let filename: string;
if (plusLine?.startsWith("+++ b/")) {
filename = plusLine.slice(6);
} else if (plusLine === "+++ /dev/null" && minusLine?.startsWith("--- a/")) {
} else if (
plusLine === "+++ /dev/null" &&
minusLine?.startsWith("--- a/")
) {
filename = minusLine.slice(6); // deleted file — use old path
} else {
continue;
@@ -113,6 +152,9 @@ export function parseDiff(rawDiff: string): DiffFile[] {
}
export function chunkFile(file: DiffFile, maxTokens: number): DiffChunk[] {
console.log(
`Chunking file ${file.filename} with ${file.hunks.length} hunks...`,
);
return file.hunks.map(({ header, body }) => {
let hunk = body;
@@ -122,13 +164,21 @@ export function chunkFile(file: DiffFile, maxTokens: number): DiffChunk[] {
const charBudget = maxTokens * 4;
let chars = 0;
let i = 0;
while (i < lines.length && chars + (lines[i]?.length ?? 0) + 1 <= charBudget) {
while (
i < lines.length &&
chars + (lines[i]?.length ?? 0) + 1 <= charBudget
) {
chars += (lines[i]?.length ?? 0) + 1;
i++;
}
hunk = lines.slice(0, i).join("\n");
}
return { filename: file.filename, language: file.language, hunk, context: header };
return {
filename: file.filename,
language: file.language,
hunk,
context: header,
};
});
}