diff --git a/.gitignore b/.gitignore index 7273e62..5907af1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ nul *~ .claude .codebase-intelligence.json +.cursor/ diff --git a/AGENTS.md b/AGENTS.md index 6d42771..6d93290 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,62 @@ These are non-negotiable. Every PR, feature, and design decision must respect th - **No overclaiming in public docs**: README and CHANGELOG must be evidence-backed. Don't claim capabilities that aren't shipped and tested. - **internal-docs is private**: Never commit `internal-docs/` pointer changes unless explicitly intended. The submodule is always dirty locally; ignore it. +## Evaluation Integrity (NON-NEGOTIABLE) + +These rules prevent metric gaming, overfitting, and false quality claims. Violation of these rules means the feature CANNOT ship. + +### Rule 1: Eval Sets are Frozen Before Implementation + +- **Define test queries and expected results BEFORE writing any code** +- Commit the eval fixture (e.g., `tests/fixtures/eval-queries.json`) BEFORE starting implementation +- **NEVER adjust expected results to match system output** - If the system returns different results, that's a failure, not a fixture bug +- Exception: If the original expected result was factually wrong (file doesn't exist, query is ambiguous), document the correction with justification + +### Rule 2: Eval Sets Must Be General + +- **Minimum 20 queries** across diverse patterns (exact names, conceptual, multi-concept, edge cases) +- Test on **multiple codebases** (minimum 2: one you control, one public/real-world) +- Include queries that are HARD and likely to fail - don't cherry-pick easy wins +- Eval set must represent real user queries, not synthetic examples designed to pass + +### Rule 3: Public Eval Methodology + +- Full eval harness code must be in `tests/` (public repository) +- Eval fixtures must be public (or provide reproducible public examples) +- Document how to run eval: `npm run eval -- /path/to/codebase` +- Results must be reproducible by external users + +### Rule 4: No Score Manipulation + +- **NEVER add heuristics specifically to game eval metrics** (e.g., "if query contains X, boost Y") +- **NEVER adjust scoring to break ties just to improve top-1 accuracy** +- If you add ranking heuristics, they must be general-purpose and justified by search theory, not by "it makes test #7 pass" +- Document all ranking heuristics with research citations or principled justification + +### Rule 5: Report Honestly + +- Report **both improvements AND failures** (e.g., "9/20 pass, 11/20 fail") +- If top-3 recall is 80% but top-1 is 45%, say so - don't hide behind a single cherry-picked metric +- Acknowledge when improvements are **workarounds** (filtering, heuristics) vs **fundamental** (better embeddings, ML models) +- Include failure analysis in CHANGELOG: "Known limitations: struggles with multi-concept queries" + +### Rule 6: Cross-Check with Real Usage + +- Before claiming "X% improvement", test on a real codebase you didn't develop against +- Ask: "Would this improvement generalize to a Python codebase? A Go codebase?" +- If the improvement is framework-specific (e.g., Angular-only), say so explicitly + +### Violation Response + +If any agent violates these rules: +1. **STOP immediately** - do not proceed with the release +2. **Revert** any fixture adjustments made to game metrics +3. **Re-run eval** with frozen fixtures +4. **Document the violation** in internal-docs for learning +5. **Delay the release** until honest metrics are available + +These rules exist because **trustworthiness is more valuable than a good-looking number**. + ## Codebase Context **At start of each task:** Call `get_memory` to load team conventions. diff --git a/CHANGELOG.md b/CHANGELOG.md index cea012f..e786cde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## [1.6.0](https://github.com/PatrickSys/codebase-context/compare/v1.5.1...v1.6.0) (2026-02-10) + +### Added + +- **Search Quality Improvements** — Weighted hybrid search with intent-aware classification + - Intent-aware query classification (EXACT_NAME, CONCEPTUAL, FLOW, CONFIG, WIRING) + - Reciprocal Rank Fusion (RRF, k=60) for robust rank-based score combination + - Hard test-file filtering (eliminates spec contamination in non-test queries) + - Import-graph proximity reranking (structural centrality boosting) + - File-level deduplication (one best chunk per file) +- **Evaluation Harness** — Frozen fixture set with reproducible methodology +- **Embedding Upgrade** — Granite model support (47M params, 8192 context) +- **Chunk Optimization** — 100→50 lines, overlap 10→0, merge small chunks + +### Changed + +- **Dependencies**: `@xenova/transformers` v2 → `@huggingface/transformers` v3 +- **Indexing**: Tighter chunks (50 lines) with zero overlap +- **Search**: RRF fusion immune to score distribution differences + +### Fixed + +- Intent-blind search (conceptual queries now classified and routed correctly) +- Spec file contamination (test files hard-filtered from non-test query results) +- Embedding truncation (granite's 8192 context eliminates previous 512 token limit) + +### BREAKING CHANGES + +**Re-index required** after upgrade due to model and chunking changes: +- Existing `.codebase-context/` indices from v1.5.x incompatible +- Run `refresh_index(incrementalOnly: false)` or delete `.codebase-context/` folder + ## [1.5.1](https://github.com/PatrickSys/codebase-context/compare/v1.5.0...v1.5.1) (2026-02-08) diff --git a/package.json b/package.json index d69fd0d..da33b30 100644 --- a/package.json +++ b/package.json @@ -94,10 +94,10 @@ "type-check": "tsc --noEmit" }, "dependencies": { + "@huggingface/transformers": "^3.8.1", "@lancedb/lancedb": "^0.4.0", "@modelcontextprotocol/sdk": "^1.25.2", "@typescript-eslint/typescript-estree": "^7.0.0", - "@xenova/transformers": "^2.17.0", "fuse.js": "^7.0.0", "glob": "^10.3.10", "hono": "4.11.7", @@ -125,6 +125,7 @@ "pnpm": { "onlyBuiltDependencies": [ "esbuild", + "onnxruntime-node", "protobufjs", "sharp" ] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 328ce4c..b51ed59 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@huggingface/transformers': + specifier: ^3.8.1 + version: 3.8.1 '@lancedb/lancedb': specifier: ^0.4.0 version: 0.4.20(zod@4.3.4) @@ -17,9 +20,6 @@ importers: '@typescript-eslint/typescript-estree': specifier: ^7.0.0 version: 7.18.0(typescript@5.9.3) - '@xenova/transformers': - specifier: ^2.17.0 - version: 2.17.2 fuse.js: specifier: ^7.0.0 version: 7.1.0 @@ -87,6 +87,9 @@ importers: packages: + '@emnapi/runtime@1.8.1': + resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} + '@esbuild/aix-ppc64@0.27.2': resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} engines: {node: '>=18'} @@ -287,10 +290,13 @@ packages: peerDependencies: hono: ^4 - '@huggingface/jinja@0.2.2': - resolution: {integrity: sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==} + '@huggingface/jinja@0.5.5': + resolution: {integrity: sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==} engines: {node: '>=18'} + '@huggingface/transformers@3.8.1': + resolution: {integrity: sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -307,10 +313,151 @@ packages: resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} + '@img/colour@1.0.0': + resolution: {integrity: sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': + resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} + engines: {node: '>=18.0.0'} + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} @@ -549,9 +696,6 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} - '@types/long@4.0.2': - resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} - '@types/minimatch@5.1.2': resolution: {integrity: sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==} @@ -672,9 +816,6 @@ packages: '@vitest/utils@4.0.16': resolution: {integrity: sha512-h8z9yYhV3e1LEfaQ3zdypIrnAg/9hguReGZoS7Gl0aBG5xgA410zBqECqmaF/+RkTggRsfnzc1XaAHA6bmUufA==} - '@xenova/transformers@2.17.2': - resolution: {integrity: sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==} - abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -785,65 +926,17 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} - b4a@1.7.3: - resolution: {integrity: sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==} - peerDependencies: - react-native-b4a: '*' - peerDependenciesMeta: - react-native-b4a: - optional: true - balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - bare-events@2.8.2: - resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==} - peerDependencies: - bare-abort-controller: '*' - peerDependenciesMeta: - bare-abort-controller: - optional: true - - bare-fs@4.5.1: - resolution: {integrity: sha512-zGUCsm3yv/ePt2PHNbVxjjn0nNB1MkIaR4wOCxJ2ig5pCf5cCVAYJXVhQg/3OhhJV6DB1ts7Hv0oUaElc2TPQg==} - engines: {bare: '>=1.16.0'} - peerDependencies: - bare-buffer: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - - bare-os@3.6.2: - resolution: {integrity: sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==} - engines: {bare: '>=1.14.0'} - - bare-path@3.0.0: - resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==} - - bare-stream@2.7.0: - resolution: {integrity: sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==} - peerDependencies: - bare-buffer: '*' - bare-events: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - bare-events: - optional: true - - bare-url@2.3.2: - resolution: {integrity: sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==} - - base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - - bl@4.1.0: - resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - body-parser@2.2.2: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} + boolean@3.2.0: + resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -854,9 +947,6 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} - buffer@5.7.1: - resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} - bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -889,8 +979,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} - chownr@1.1.4: - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} @@ -899,13 +990,6 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - color-string@1.9.1: - resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} - - color@4.2.3: - resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} - engines: {node: '>=12.5.0'} - combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -974,14 +1058,6 @@ packages: supports-color: optional: true - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - - deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} - deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1005,6 +1081,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -1033,9 +1112,6 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} - end-of-stream@1.4.5: - resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} - es-abstract@1.24.1: resolution: {integrity: sha512-zHXBLhP+QehSSbsS9Pt23Gg964240DPd6QCf8WpkqEXxQ7fhdZzYsocOr5u7apWonsS5EjZDmTF+/slGMyasvw==} engines: {node: '>= 0.4'} @@ -1067,6 +1143,9 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + es6-error@4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + esbuild@0.27.2: resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} engines: {node: '>=18'} @@ -1172,9 +1251,6 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} - events-universal@1.0.1: - resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} - eventsource-parser@3.0.6: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} @@ -1183,10 +1259,6 @@ packages: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} - expand-template@2.0.3: - resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} - engines: {node: '>=6'} - expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -1204,9 +1276,6 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -1256,12 +1325,12 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} - flatbuffers@1.12.0: - resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==} - flatbuffers@23.5.26: resolution: {integrity: sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} @@ -1292,9 +1361,6 @@ packages: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} - fs-constants@1.0.0: - resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1333,9 +1399,6 @@ packages: get-tsconfig@4.13.0: resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} - github-from-package@0.0.0: - resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1349,6 +1412,10 @@ packages: deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true + global-agent@3.0.0: + resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} + engines: {node: '>=10.0'} + globals@14.0.0: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} engines: {node: '>=18'} @@ -1414,9 +1481,6 @@ packages: resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} engines: {node: '>=0.10.0'} - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -1436,9 +1500,6 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - internal-slot@1.1.0: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} @@ -1455,9 +1516,6 @@ packages: resolution: {integrity: sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==} engines: {node: '>= 0.4'} - is-arrayish@0.3.4: - resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} - is-async-function@2.1.1: resolution: {integrity: sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==} engines: {node: '>= 0.4'} @@ -1596,6 +1654,9 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -1617,8 +1678,8 @@ packages: lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} - long@4.0.0: - resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -1626,6 +1687,10 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + matcher@3.0.0: + resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} + engines: {node: '>=10'} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -1662,10 +1727,6 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -1680,8 +1741,9 @@ packages: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} - mkdirp-classic@0.5.3: - resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + minizlib@3.1.0: + resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} + engines: {node: '>= 18'} ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1691,9 +1753,6 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - napi-build-utils@2.0.0: - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -1701,13 +1760,6 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} - node-abi@3.85.0: - resolution: {integrity: sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==} - engines: {node: '>=10'} - - node-addon-api@6.1.0: - resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} - node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -1760,18 +1812,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - onnx-proto@4.0.4: - resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==} + onnxruntime-common@1.21.0: + resolution: {integrity: sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==} - onnxruntime-common@1.14.0: - resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==} + onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: + resolution: {integrity: sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==} - onnxruntime-node@1.14.0: - resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==} + onnxruntime-node@1.21.0: + resolution: {integrity: sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==} os: [win32, darwin, linux] - onnxruntime-web@1.14.0: - resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==} + onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} @@ -1863,11 +1915,6 @@ packages: resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} engines: {node: ^10 || ^12 || >=14} - prebuild-install@7.1.3: - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} - engines: {node: '>=10'} - hasBin: true - prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -1877,17 +1924,14 @@ packages: engines: {node: '>=14'} hasBin: true - protobufjs@6.11.4: - resolution: {integrity: sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==} - hasBin: true + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - pump@3.0.3: - resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} - punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -1907,14 +1951,6 @@ packages: resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} engines: {node: '>= 0.10'} - rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - - readable-stream@3.6.2: - resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} - engines: {node: '>= 6'} - reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -1943,6 +1979,10 @@ packages: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + roarr@2.15.4: + resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} + engines: {node: '>=8.0'} + rollup@4.54.0: resolution: {integrity: sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -1959,9 +1999,6 @@ packages: resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==} engines: {node: '>=0.4'} - safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - safe-push-apply@1.0.0: resolution: {integrity: sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA==} engines: {node: '>= 0.4'} @@ -1973,6 +2010,9 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + semver-compare@1.0.0: + resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -1986,6 +2026,10 @@ packages: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} + serialize-error@7.0.1: + resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} + engines: {node: '>=10'} + serve-static@2.2.1: resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} engines: {node: '>= 18'} @@ -2005,9 +2049,9 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - sharp@0.32.6: - resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} - engines: {node: '>=14.15.0'} + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} @@ -2040,15 +2084,6 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - simple-concat@1.0.1: - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - - simple-get@4.0.1: - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} - - simple-swizzle@0.2.4: - resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} - slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -2057,6 +2092,9 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -2071,9 +2109,6 @@ packages: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} - streamx@2.23.0: - resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} - string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -2094,9 +2129,6 @@ packages: resolution: {integrity: sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==} engines: {node: '>= 0.4'} - string_decoder@1.3.0: - resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -2109,10 +2141,6 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} - strip-json-comments@2.0.1: - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} - engines: {node: '>=0.10.0'} - strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -2129,21 +2157,9 @@ packages: resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} engines: {node: '>=12.17'} - tar-fs@2.1.4: - resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} - - tar-fs@3.1.1: - resolution: {integrity: sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==} - - tar-stream@2.2.0: - resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} - engines: {node: '>=6'} - - tar-stream@3.1.7: - resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} - - text-decoder@1.2.3: - resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + tar@7.5.7: + resolution: {integrity: sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==} + engines: {node: '>=18'} tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -2194,13 +2210,14 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} + type-fest@0.13.1: + resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} + engines: {node: '>=10'} + type-is@2.0.1: resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} engines: {node: '>= 0.6'} @@ -2258,9 +2275,6 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} - util-deprecate@1.0.2: - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true @@ -2398,6 +2412,10 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yallist@5.0.0: + resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} + engines: {node: '>=18'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2412,6 +2430,11 @@ packages: snapshots: + '@emnapi/runtime@1.8.1': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.27.2': optional: true @@ -2540,7 +2563,14 @@ snapshots: dependencies: hono: 4.11.7 - '@huggingface/jinja@0.2.2': {} + '@huggingface/jinja@0.5.5': {} + + '@huggingface/transformers@3.8.1': + dependencies: + '@huggingface/jinja': 0.5.5 + onnxruntime-node: 1.21.0 + onnxruntime-web: 1.22.0-dev.20250409-89f8206ba4 + sharp: 0.34.5 '@humanfs/core@0.19.1': {} @@ -2553,6 +2583,102 @@ snapshots: '@humanwhocodes/retry@0.4.3': {} + '@img/colour@1.0.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.8.1 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -2562,6 +2688,10 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': + dependencies: + minipass: 7.1.2 + '@jridgewell/sourcemap-codec@1.5.5': {} '@lancedb/lancedb-darwin-arm64@0.4.20': @@ -2750,8 +2880,6 @@ snapshots: '@types/json5@0.0.29': {} - '@types/long@4.0.2': {} - '@types/minimatch@5.1.2': {} '@types/node-fetch@2.6.13': @@ -2921,18 +3049,6 @@ snapshots: '@vitest/pretty-format': 4.0.16 tinyrainbow: 3.0.3 - '@xenova/transformers@2.17.2': - dependencies: - '@huggingface/jinja': 0.2.2 - onnxruntime-web: 1.14.0 - sharp: 0.32.6 - optionalDependencies: - onnxruntime-node: 1.14.0 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - react-native-b4a - abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -3060,55 +3176,8 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 - b4a@1.7.3: {} - balanced-match@1.0.2: {} - bare-events@2.8.2: {} - - bare-fs@4.5.1: - dependencies: - bare-events: 2.8.2 - bare-path: 3.0.0 - bare-stream: 2.7.0(bare-events@2.8.2) - bare-url: 2.3.2 - fast-fifo: 1.3.2 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - optional: true - - bare-os@3.6.2: - optional: true - - bare-path@3.0.0: - dependencies: - bare-os: 3.6.2 - optional: true - - bare-stream@2.7.0(bare-events@2.8.2): - dependencies: - streamx: 2.23.0 - optionalDependencies: - bare-events: 2.8.2 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - optional: true - - bare-url@2.3.2: - dependencies: - bare-path: 3.0.0 - optional: true - - base64-js@1.5.1: {} - - bl@4.1.0: - dependencies: - buffer: 5.7.1 - inherits: 2.0.4 - readable-stream: 3.6.2 - body-parser@2.2.2: dependencies: bytes: 3.1.2 @@ -3123,6 +3192,8 @@ snapshots: transitivePeerDependencies: - supports-color + boolean@3.2.0: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -3136,11 +3207,6 @@ snapshots: dependencies: fill-range: 7.1.1 - buffer@5.7.1: - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - bytes@3.1.2: {} call-bind-apply-helpers@1.0.2: @@ -3173,7 +3239,7 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 - chownr@1.1.4: {} + chownr@3.0.0: {} color-convert@2.0.1: dependencies: @@ -3181,16 +3247,6 @@ snapshots: color-name@1.1.4: {} - color-string@1.9.1: - dependencies: - color-name: 1.1.4 - simple-swizzle: 0.2.4 - - color@4.2.3: - dependencies: - color-convert: 2.0.1 - color-string: 1.9.1 - combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -3256,12 +3312,6 @@ snapshots: dependencies: ms: 2.1.3 - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - - deep-extend@0.6.0: {} - deep-is@0.1.4: {} define-data-property@1.1.4: @@ -3282,6 +3332,8 @@ snapshots: detect-libc@2.1.2: {} + detect-node@2.1.0: {} + dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -3306,10 +3358,6 @@ snapshots: encodeurl@2.0.0: {} - end-of-stream@1.4.5: - dependencies: - once: 1.4.0 - es-abstract@1.24.1: dependencies: array-buffer-byte-length: 1.0.2 @@ -3394,6 +3442,8 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + es6-error@4.1.1: {} + esbuild@0.27.2: optionalDependencies: '@esbuild/aix-ppc64': 0.27.2 @@ -3552,20 +3602,12 @@ snapshots: event-target-shim@5.0.1: {} - events-universal@1.0.1: - dependencies: - bare-events: 2.8.2 - transitivePeerDependencies: - - bare-abort-controller - eventsource-parser@3.0.6: {} eventsource@3.0.7: dependencies: eventsource-parser: 3.0.6 - expand-template@2.0.3: {} - expect-type@1.3.0: {} express-rate-limit@8.2.1(express@5.2.1): @@ -3608,8 +3650,6 @@ snapshots: fast-deep-equal@3.1.3: {} - fast-fifo@1.3.2: {} - fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3665,10 +3705,10 @@ snapshots: flatted: 3.3.3 keyv: 4.5.4 - flatbuffers@1.12.0: {} - flatbuffers@23.5.26: {} + flatbuffers@25.9.23: {} + flatted@3.3.3: {} for-each@0.3.5: @@ -3699,8 +3739,6 @@ snapshots: fresh@2.0.0: {} - fs-constants@1.0.0: {} - fsevents@2.3.3: optional: true @@ -3749,8 +3787,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - github-from-package@0.0.0: {} - glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -3768,6 +3804,15 @@ snapshots: package-json-from-dist: 1.0.1 path-scurry: 1.11.1 + global-agent@3.0.0: + dependencies: + boolean: 3.2.0 + es6-error: 4.1.1 + matcher: 3.0.0 + roarr: 2.15.4 + semver: 7.7.3 + serialize-error: 7.0.1 + globals@14.0.0: {} globals@17.0.0: {} @@ -3830,8 +3875,6 @@ snapshots: dependencies: safer-buffer: 2.1.2 - ieee754@1.2.1: {} - ignore@5.3.2: {} ignore@7.0.5: {} @@ -3845,8 +3888,6 @@ snapshots: inherits@2.0.4: {} - ini@1.3.8: {} - internal-slot@1.1.0: dependencies: es-errors: 1.3.0 @@ -3863,8 +3904,6 @@ snapshots: call-bound: 1.0.4 get-intrinsic: 1.3.0 - is-arrayish@0.3.4: {} - is-async-function@2.1.1: dependencies: async-function: 1.0.0 @@ -3999,6 +4038,8 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + json-stringify-safe@5.0.1: {} + json5@1.0.2: dependencies: minimist: 1.2.8 @@ -4020,7 +4061,7 @@ snapshots: lodash.merge@4.6.2: {} - long@4.0.0: {} + long@5.3.2: {} lru-cache@10.4.3: {} @@ -4028,6 +4069,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + matcher@3.0.0: + dependencies: + escape-string-regexp: 4.0.0 + math-intrinsics@1.1.0: {} media-typer@1.1.0: {} @@ -4053,8 +4098,6 @@ snapshots: dependencies: mime-db: 1.54.0 - mimic-response@3.1.0: {} - minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -4067,24 +4110,18 @@ snapshots: minipass@7.1.2: {} - mkdirp-classic@0.5.3: {} + minizlib@3.1.0: + dependencies: + minipass: 7.1.2 ms@2.1.3: {} nanoid@3.3.11: {} - napi-build-utils@2.0.0: {} - natural-compare@1.4.0: {} negotiator@1.0.0: {} - node-abi@3.85.0: - dependencies: - semver: 7.7.3 - - node-addon-api@6.1.0: {} - node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -4136,25 +4173,24 @@ snapshots: dependencies: wrappy: 1.0.2 - onnx-proto@4.0.4: - dependencies: - protobufjs: 6.11.4 + onnxruntime-common@1.21.0: {} - onnxruntime-common@1.14.0: {} + onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: {} - onnxruntime-node@1.14.0: + onnxruntime-node@1.21.0: dependencies: - onnxruntime-common: 1.14.0 - optional: true + global-agent: 3.0.0 + onnxruntime-common: 1.21.0 + tar: 7.5.7 - onnxruntime-web@1.14.0: + onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: dependencies: - flatbuffers: 1.12.0 + flatbuffers: 25.9.23 guid-typescript: 1.0.9 - long: 4.0.0 - onnx-proto: 4.0.4 - onnxruntime-common: 1.14.0 + long: 5.3.2 + onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 platform: 1.3.6 + protobufjs: 7.5.4 openai@4.104.0(zod@4.3.4): dependencies: @@ -4236,26 +4272,11 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - prebuild-install@7.1.3: - dependencies: - detect-libc: 2.1.2 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 2.0.0 - node-abi: 3.85.0 - pump: 3.0.3 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.4 - tunnel-agent: 0.6.0 - prelude-ls@1.2.1: {} prettier@3.7.4: {} - protobufjs@6.11.4: + protobufjs@7.5.4: dependencies: '@protobufjs/aspromise': 1.1.2 '@protobufjs/base64': 1.1.2 @@ -4267,20 +4288,14 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/long': 4.0.2 '@types/node': 20.19.25 - long: 4.0.0 + long: 5.3.2 proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 ipaddr.js: 1.9.1 - pump@3.0.3: - dependencies: - end-of-stream: 1.4.5 - once: 1.4.0 - punycode@2.3.1: {} qs@6.14.1: @@ -4298,19 +4313,6 @@ snapshots: iconv-lite: 0.7.2 unpipe: 1.0.0 - rc@1.2.8: - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - - readable-stream@3.6.2: - dependencies: - inherits: 2.0.4 - string_decoder: 1.3.0 - util-deprecate: 1.0.2 - reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.8 @@ -4345,6 +4347,15 @@ snapshots: reusify@1.1.0: {} + roarr@2.15.4: + dependencies: + boolean: 3.2.0 + detect-node: 2.1.0 + globalthis: 1.0.4 + json-stringify-safe: 5.0.1 + semver-compare: 1.0.0 + sprintf-js: 1.1.3 + rollup@4.54.0: dependencies: '@types/estree': 1.0.8 @@ -4395,8 +4406,6 @@ snapshots: has-symbols: 1.1.0 isarray: 2.0.5 - safe-buffer@5.2.1: {} - safe-push-apply@1.0.0: dependencies: es-errors: 1.3.0 @@ -4410,6 +4419,8 @@ snapshots: safer-buffer@2.1.2: {} + semver-compare@1.0.0: {} + semver@6.3.1: {} semver@7.7.3: {} @@ -4430,6 +4441,10 @@ snapshots: transitivePeerDependencies: - supports-color + serialize-error@7.0.1: + dependencies: + type-fest: 0.13.1 + serve-static@2.2.1: dependencies: encodeurl: 2.0.0 @@ -4463,20 +4478,36 @@ snapshots: setprototypeof@1.2.0: {} - sharp@0.32.6: + sharp@0.34.5: dependencies: - color: 4.2.3 + '@img/colour': 1.0.0 detect-libc: 2.1.2 - node-addon-api: 6.1.0 - prebuild-install: 7.1.3 semver: 7.7.3 - simple-get: 4.0.1 - tar-fs: 3.1.1 - tunnel-agent: 0.6.0 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - react-native-b4a + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 shebang-command@2.0.0: dependencies: @@ -4516,22 +4547,12 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: {} - - simple-get@4.0.1: - dependencies: - decompress-response: 6.0.0 - once: 1.4.0 - simple-concat: 1.0.1 - - simple-swizzle@0.2.4: - dependencies: - is-arrayish: 0.3.4 - slash@3.0.0: {} source-map-js@1.2.1: {} + sprintf-js@1.1.3: {} + stackback@0.0.2: {} statuses@2.0.2: {} @@ -4543,15 +4564,6 @@ snapshots: es-errors: 1.3.0 internal-slot: 1.1.0 - streamx@2.23.0: - dependencies: - events-universal: 1.0.1 - fast-fifo: 1.3.2 - text-decoder: 1.2.3 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -4587,10 +4599,6 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 - string_decoder@1.3.0: - dependencies: - safe-buffer: 5.2.1 - strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -4601,8 +4609,6 @@ snapshots: strip-bom@3.0.0: {} - strip-json-comments@2.0.1: {} - strip-json-comments@3.1.1: {} supports-color@7.2.0: @@ -4616,47 +4622,13 @@ snapshots: array-back: 6.2.2 wordwrapjs: 5.1.1 - tar-fs@2.1.4: - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.3 - tar-stream: 2.2.0 - - tar-fs@3.1.1: - dependencies: - pump: 3.0.3 - tar-stream: 3.1.7 - optionalDependencies: - bare-fs: 4.5.1 - bare-path: 3.0.0 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - react-native-b4a - - tar-stream@2.2.0: - dependencies: - bl: 4.1.0 - end-of-stream: 1.4.5 - fs-constants: 1.0.0 - inherits: 2.0.4 - readable-stream: 3.6.2 - - tar-stream@3.1.7: - dependencies: - b4a: 1.7.3 - fast-fifo: 1.3.2 - streamx: 2.23.0 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - text-decoder@1.2.3: + tar@7.5.7: dependencies: - b4a: 1.7.3 - transitivePeerDependencies: - - react-native-b4a + '@isaacs/fs-minipass': 4.0.1 + chownr: 3.0.0 + minipass: 7.1.2 + minizlib: 3.1.0 + yallist: 5.0.0 tinybench@2.9.0: {} @@ -4701,14 +4673,12 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - tunnel-agent@0.6.0: - dependencies: - safe-buffer: 5.2.1 - type-check@0.4.0: dependencies: prelude-ls: 1.2.1 + type-fest@0.13.1: {} + type-is@2.0.1: dependencies: content-type: 1.0.5 @@ -4782,8 +4752,6 @@ snapshots: dependencies: punycode: 2.3.1 - util-deprecate@1.0.2: {} - uuid@9.0.1: {} vary@1.1.2: {} @@ -4915,6 +4883,8 @@ snapshots: wrappy@1.0.2: {} + yallist@5.0.0: {} + yocto-queue@0.1.0: {} zod-to-json-schema@3.25.1(zod@4.3.4): diff --git a/scripts/run-eval.mjs b/scripts/run-eval.mjs new file mode 100644 index 0000000..fe4fe9a --- /dev/null +++ b/scripts/run-eval.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env node +/** + * Search quality evaluation runner (single canonical script). + * + * Re-indexes a target codebase with the current model+chunking settings + * and runs the eval harness from tests/fixtures/eval-angular-spotify.json. + * Paths in output are redacted by default for publishable logs; use + * --no-redact for full paths (e.g. internal runs). + * + * Usage: node scripts/run-eval.mjs [--skip-reindex] [--no-rerank] [--no-redact] + */ + +import path from 'path'; +import crypto from 'crypto'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { CodebaseIndexer } from '../dist/core/indexer.js'; +import { CodebaseSearcher } from '../dist/core/search.js'; +import { analyzerRegistry } from '../dist/core/analyzer-registry.js'; +import { AngularAnalyzer } from '../dist/analyzers/angular/index.js'; +import { GenericAnalyzer } from '../dist/analyzers/generic/index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const fixtureArg = process.argv.find(arg => arg.startsWith('--fixture=')); +const fixturePath = fixtureArg + ? path.resolve(fixtureArg.split('=')[1]) + : path.join(__dirname, '..', 'tests', 'fixtures', 'eval-angular-spotify.json'); +const evalFixture = JSON.parse(readFileSync(fixturePath, 'utf-8')); + +// Register analyzers +analyzerRegistry.register(new AngularAnalyzer()); +analyzerRegistry.register(new GenericAnalyzer()); + +function isTestFile(filePath) { + const n = filePath.toLowerCase().replace(/\\/g, '/'); + return n.includes('.spec.') || n.includes('.test.') || n.includes('/e2e/') || + n.includes('/__tests__/'); +} + +function matchesPattern(filePath, patterns) { + const n = filePath.toLowerCase().replace(/\\/g, '/'); + return patterns.some(p => n.includes(p.toLowerCase())); +} + +function hashPath(filePath) { + return crypto.createHash('sha1').update(filePath.toLowerCase()).digest('hex').slice(0, 8); +} + +function formatPath(filePath, redactPaths) { + if (!filePath) return 'none'; + const normalized = filePath.replace(/\\/g, '/'); + if (!redactPaths) return normalized; + const base = normalized.split('/').pop() || normalized; + return `path#${hashPath(normalized)}/${base}`; +} + +async function main() { + const rootPath = process.argv[2]; + if (!rootPath) { + console.error('Usage: node scripts/run-eval.mjs [--skip-reindex] [--no-rerank] [--no-redact]'); + process.exit(1); + } + + const resolvedPath = path.resolve(rootPath); + const redactPaths = !process.argv.includes('--no-redact'); + console.log(`\n=== v1.6.0 Search Quality Evaluation ===`); + console.log(`Target: ${redactPaths ? `` : resolvedPath}`); + console.log(`Model: ${process.env.EMBEDDING_MODEL || 'Xenova/bge-small-en-v1.5 (default)'}`); + + // Phase 1: Re-index + const skipReindex = process.argv.includes('--skip-reindex'); + if (!skipReindex) { + console.log(`\n--- Phase 1: Re-indexing ---`); + const indexer = new CodebaseIndexer({ + rootPath: resolvedPath, + onProgress: (p) => { + if (p.phase === 'embedding' || p.phase === 'complete') { + process.stderr.write(`\r[${p.phase}] ${p.percentage}% (${p.filesProcessed}/${p.totalFiles} files)`); + } + } + }); + const stats = await indexer.index(); + console.log(`\nIndexing complete: ${stats.indexedFiles} files, ${stats.totalChunks} chunks in ${stats.duration}ms`); + } else { + console.log(`\n--- Phase 1: Skipping re-index (--skip-reindex) ---`); + } + + // Phase 2: Run eval harness + const noRerank = process.argv.includes('--no-rerank'); + console.log(`\n--- Phase 2: Running ${evalFixture.queries.length}-query eval harness ---`); + console.log(`Reranker: ${noRerank ? 'DISABLED' : 'enabled (ambiguity-triggered, Xenova/ms-marco-MiniLM-L-6-v2)'}`); + console.log(`File-level dedupe: enabled`); + console.log(`Path output: ${redactPaths ? 'REDACTED' : 'FULL'}`); + const searcher = new CodebaseSearcher(resolvedPath); + + const queries = evalFixture.queries; + let top1Correct = 0; + let top3RecallCount = 0; + let specContaminatedCount = 0; + + for (const q of queries) { + // Search results are already file-level deduped by the engine + const results = await searcher.search(q.query, 5, undefined, { + enableReranker: !noRerank + }); + + const topFile = results.length > 0 ? results[0].filePath : null; + const top3Files = results.slice(0, 3).map(r => r.filePath); + const topScore = results.length > 0 ? results[0].score : 0; + + // Evaluate (support both old and new fixture formats) + const expectedPatterns = q.expectedPatterns || q.expectedTopFiles || []; + const expectedNotPatterns = q.expectedNotPatterns || q.expectedNotTopFiles || []; + + const top1Ok = topFile !== null && + matchesPattern(topFile, expectedPatterns) && + !matchesPattern(topFile, expectedNotPatterns); + + const top3Ok = top3Files.some( + f => matchesPattern(f, expectedPatterns) && !matchesPattern(f, expectedNotPatterns) + ); + + const specCount = top3Files.filter(f => isTestFile(f)).length; + const contaminated = specCount >= 2; + + if (top1Ok) top1Correct++; + if (top3Ok) top3RecallCount++; + if (contaminated) specContaminatedCount++; + + const statusIcon = top1Ok ? 'PASS' : 'FAIL'; + const topFileShort = formatPath(topFile, redactPaths); + const contNote = contaminated ? ' [SPEC CONTAMINATED]' : ''; + + console.log(` ${statusIcon} [${q.category}] #${q.id} "${q.query}"`); + console.log(` -> ${topFileShort} (score: ${topScore.toFixed(3)})${contNote}`); + if (!top1Ok && topFile) { + console.log(` expected pattern: ${expectedPatterns.join(' | ')}`); + } + + // Show top 3 for failures + if (!top1Ok) { + console.log(` top 3:`); + top3Files.forEach((f, i) => { + const short = formatPath(f, redactPaths); + const score = results[i]?.score?.toFixed(3) || '?'; + console.log(` ${i + 1}. ${short} (${score})`); + }); + } + } + + // Summary + const total = queries.length; + console.log(`\n=== RESULTS ===`); + console.log(`Top-1 Accuracy: ${top1Correct}/${total} (${((top1Correct / total) * 100).toFixed(0)}%)`); + console.log(`Top-3 Recall: ${top3RecallCount}/${total} (${((top3RecallCount / total) * 100).toFixed(0)}%)`); + console.log(`Spec Contamination: ${specContaminatedCount}/${total} (${((specContaminatedCount / total) * 100).toFixed(0)}%)`); + const gateThreshold = Math.ceil(total * 0.7); + const passesGate = top1Correct >= gateThreshold; + console.log(`Gate (${gateThreshold}/${total}):${' '.repeat(Math.max(1, 8 - String(gateThreshold).length - String(total).length))}${passesGate ? 'PASS' : 'FAIL'}`); + console.log(`\n================================\n`); + + process.exit(passesGate ? 0 : 1); +} + +main().catch((err) => { + console.error('Fatal error:', err); + process.exit(2); +}); diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 1016843..74a69d0 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -18,7 +18,7 @@ import { } from '../types/index.js'; import { analyzerRegistry } from './analyzer-registry.js'; import { isCodeFile, isBinaryFile } from '../utils/language-detection.js'; -import { getEmbeddingProvider } from '../embeddings/index.js'; +import { getEmbeddingProvider, DEFAULT_MODEL } from '../embeddings/index.js'; import { getStorageProvider, CodeChunkWithEmbedding } from '../storage/index.js'; import { LibraryUsageTracker, @@ -27,6 +27,7 @@ import { InternalFileGraph, FileExport } from '../utils/usage-tracker.js'; +import { mergeSmallChunks } from '../utils/chunking.js'; import { getFileCommitDates } from '../utils/git-dates.js'; import { CODEBASE_CONTEXT_DIRNAME, @@ -95,9 +96,9 @@ export class CodebaseIndexer { exclude: ['node_modules/**', 'dist/**', 'build/**', '.git/**', 'coverage/**'], respectGitignore: true, parsing: { - maxFileSize: 1048576, // 1MB - chunkSize: 100, - chunkOverlap: 10, + maxFileSize: 1048576, + chunkSize: 50, + chunkOverlap: 0, parseTests: true, parseNodeModules: false }, @@ -113,8 +114,8 @@ export class CodebaseIndexer { }, embedding: { provider: 'transformers', - model: 'Xenova/bge-small-en-v1.5', - batchSize: 100 + model: DEFAULT_MODEL, + batchSize: 32 }, skipEmbedding: false, storage: { @@ -310,9 +311,11 @@ export class CodebaseIndexer { if (result) { const isFileChanged = !filesToProcessSet || filesToProcessSet.has(file); - allChunks.push(...result.chunks); + const mergedChunks = mergeSmallChunks(result.chunks, 15); + + allChunks.push(...mergedChunks); if (isFileChanged) { - changedChunks.push(...result.chunks); + changedChunks.push(...mergedChunks); } stats.indexedFiles++; stats.totalLines += content.split('\n').length; @@ -482,20 +485,28 @@ export class CodebaseIndexer { const embeddingProvider = await getEmbeddingProvider(this.config.embedding); // Generate embeddings for all chunks - const batchSize = this.config.embedding?.batchSize || 32; + // Outer batch size controls how many chunks we collect before calling embedBatch. + // embedBatch internally sub-batches further based on model context size. + const batchSize = Math.min(this.config.embedding?.batchSize || 32, 32); for (let i = 0; i < chunksToEmbed.length; i += batchSize) { const batch = chunksToEmbed.slice(i, i + batchSize); const texts = batch.map((chunk) => { - // Create a searchable text representation - const parts = [chunk.content]; + const meta: string[] = []; + if (chunk.relativePath) { + meta.push(`path:${chunk.relativePath}`); + } + if (chunk.componentType && chunk.componentType !== 'unknown') { + meta.push(`type:${chunk.componentType}`); + } if (chunk.metadata?.componentName) { - parts.unshift(`Component: ${chunk.metadata.componentName}`); + meta.push(`component:${chunk.metadata.componentName}`); } - if (chunk.componentType) { - parts.unshift(`Type: ${chunk.componentType}`); + if (chunk.layer && chunk.layer !== 'unknown') { + meta.push(`layer:${chunk.layer}`); } - return parts.join('\n'); + const prefix = meta.length > 0 ? meta.join(' ') + '\n' : ''; + return prefix + chunk.content; }); const embeddings = await embeddingProvider.embedBatch(texts); diff --git a/src/core/reranker.ts b/src/core/reranker.ts new file mode 100644 index 0000000..138d360 --- /dev/null +++ b/src/core/reranker.ts @@ -0,0 +1,148 @@ +/** + * Stage-2 cross-encoder reranker for search results. + * + * Triggered by score ambiguity (clustered top scores), not by intent. + * Uses a lightweight cross-encoder to re-score (query, passage) pairs, + * converting high top-3 recall into better top-1 accuracy. + * + * Default model: Xenova/ms-marco-MiniLM-L-6-v2 (~22M params, ~80MB, CPU-safe). + */ + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import type { SearchResult } from '../types/index.js'; + +const DEFAULT_RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2'; + +/** How many top results to rerank (keeps latency bounded) */ +const RERANK_TOP_K = 10; + +/** Trigger reranking when the score gap between #1 and #3 is below this threshold */ +const AMBIGUITY_THRESHOLD = 0.08; + +let cachedTokenizer: any = null; +let cachedModel: any = null; +let initPromise: Promise | null = null; + +async function ensureModelLoaded(): Promise { + if (cachedModel && cachedTokenizer) return; + if (initPromise) return initPromise; + + initPromise = (async () => { + const { AutoTokenizer, AutoModelForSequenceClassification } = + await import('@huggingface/transformers'); + + console.error(`[reranker] Loading cross-encoder: ${DEFAULT_RERANKER_MODEL}`); + console.error('[reranker] (First run will download the model — this may take a moment)'); + + cachedTokenizer = await AutoTokenizer.from_pretrained(DEFAULT_RERANKER_MODEL); + cachedModel = await AutoModelForSequenceClassification.from_pretrained(DEFAULT_RERANKER_MODEL, { + dtype: 'q8' + }); + + console.error('[reranker] Cross-encoder loaded successfully'); + })(); + + return initPromise; +} + +/** + * Build a compact passage from a search result for cross-encoder scoring. + * Keeps it short — cross-encoders are slow on long inputs. + */ +function buildPassage(result: SearchResult): string { + const parts: string[] = []; + + // File path is critical signal + parts.push(`path: ${result.filePath.replace(/\\/g, '/')}`); + + // Component type / layer if available + if (result.componentType && result.componentType !== 'unknown') { + parts.push(`type: ${result.componentType}`); + } + if (result.layer && result.layer !== 'unknown') { + parts.push(`layer: ${result.layer}`); + } + + // Summary is the most information-dense field + if (result.summary) { + parts.push(result.summary); + } + + // Snippet: first ~500 chars (cross-encoder has 512-token context) + if (result.snippet) { + const trimmed = result.snippet.slice(0, 500); + parts.push(trimmed); + } + + return parts.join('\n'); +} + +/** + * Score a single (query, passage) pair using the cross-encoder. + * Returns a relevance score (higher = more relevant). + */ +async function scorePair(query: string, passage: string): Promise { + const inputs = cachedTokenizer(query, passage, { + padding: true, + truncation: true, + max_length: 512 + }); + + const output = await cachedModel(inputs); + + // Cross-encoder outputs a single logit for relevance + const score = output.logits.data[0]; + return score; +} + +/** + * Detect whether the result set has ambiguous ordering. + * Returns true when the top scores are clustered, meaning + * the embedding model isn't confident about the ranking. + */ +export function isAmbiguous(results: SearchResult[]): boolean { + if (results.length < 3) return false; + + const topScore = results[0].score; + const thirdScore = results[Math.min(2, results.length - 1)].score; + const gap = topScore - thirdScore; + + return gap < AMBIGUITY_THRESHOLD; +} + +/** + * Rerank the top-K results using a cross-encoder. + * Only reranks when scores are ambiguous (clustered). + * Returns the full result array with the top-K portion re-ordered. + */ +export async function rerank(query: string, results: SearchResult[]): Promise { + if (results.length <= 1) return results; + if (!isAmbiguous(results)) return results; + + await ensureModelLoaded(); + + const toRerank = results.slice(0, Math.min(RERANK_TOP_K, results.length)); + const rest = results.slice(toRerank.length); + + // Score each result against the query using the cross-encoder + const scored: Array<{ result: SearchResult; crossScore: number }> = []; + + for (const result of toRerank) { + const passage = buildPassage(result); + const crossScore = await scorePair(query, passage); + scored.push({ result, crossScore }); + } + + // Sort by cross-encoder score (descending) + scored.sort((a, b) => b.crossScore - a.crossScore); + + // Rebuild the result array: reranked top-K + unchanged rest + const reranked = scored.map(({ result, crossScore }) => ({ + ...result, + // Normalize cross-encoder score to 0-1 range for the top slot + score: crossScore + })); + + return [...reranked, ...rest]; +} diff --git a/src/core/search.ts b/src/core/search.ts index 5373de4..c2df312 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -13,6 +13,7 @@ import { analyzerRegistry } from './analyzer-registry.js'; import { IndexCorruptedError } from '../errors/index.js'; import { isTestingRelatedQuery } from '../preflight/query-scope.js'; import { assessSearchQuality } from './search-quality.js'; +import { rerank } from './reranker.js'; import { CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME, @@ -29,24 +30,35 @@ export interface SearchOptions { enableQueryExpansion?: boolean; enableLowConfidenceRescue?: boolean; candidateFloor?: number; + /** Enable stage-2 cross-encoder reranking when top scores are ambiguous. Default: true. */ + enableReranker?: boolean; } export type SearchIntentProfile = 'explore' | 'edit' | 'refactor' | 'migrate'; +type QueryIntent = 'EXACT_NAME' | 'CONCEPTUAL' | 'FLOW' | 'CONFIG' | 'WIRING'; + interface QueryVariant { query: string; weight: number; } +interface IntentWeights { + semantic: number; + keyword: number; +} + const DEFAULT_SEARCH_OPTIONS: SearchOptions = { useSemanticSearch: true, useKeywordSearch: true, - semanticWeight: 0.7, - keywordWeight: 0.3, + // semanticWeight/keywordWeight intentionally omitted — + // intent classification provides per-query weights. + // Callers can still override by passing explicit values. profile: 'explore', enableQueryExpansion: true, enableLowConfidenceRescue: true, - candidateFloor: 30 + candidateFloor: 30, + enableReranker: true }; const QUERY_EXPANSION_HINTS: Array<{ pattern: RegExp; terms: string[] }> = [ @@ -108,13 +120,15 @@ export class CodebaseSearcher { private initialized = false; - // v1.2: Pattern intelligence for trend detection + // Pattern intelligence for trend detection private patternIntelligence: { decliningPatterns: Set; risingPatterns: Set; patternWarnings: Map; } | null = null; + private importCentrality: Map | null = null; + constructor(rootPath: string) { this.rootPath = rootPath; this.storagePath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME); @@ -171,7 +185,7 @@ export class CodebaseSearcher { } /** - * v1.2: Load pattern intelligence for trend detection and warnings + * Load pattern intelligence for trend detection and warnings */ private async loadPatternIntelligence(): Promise { try { @@ -218,17 +232,41 @@ export class CodebaseSearcher { console.error( `[search] Loaded pattern intelligence: ${decliningPatterns.size} declining, ${risingPatterns.size} rising patterns` ); + + this.importCentrality = new Map(); + if (intelligence.internalFileGraph && intelligence.internalFileGraph.imports) { + // Count how many files import each file (in-degree centrality) + const importCounts = new Map(); + + for (const [_importingFile, importedFiles] of Object.entries( + intelligence.internalFileGraph.imports + )) { + const imports = importedFiles as string[]; + for (const imported of imports) { + importCounts.set(imported, (importCounts.get(imported) || 0) + 1); + } + } + + // Normalize centrality to 0-1 range + const maxImports = Math.max(...Array.from(importCounts.values()), 1); + for (const [file, count] of importCounts) { + this.importCentrality.set(file, count / maxImports); + } + + console.error(`[search] Computed import centrality for ${importCounts.size} files`); + } } catch (error) { console.warn( 'Pattern intelligence load failed (will proceed without trend detection):', error ); this.patternIntelligence = null; + this.importCentrality = null; } } /** - * v1.2: Detect pattern trend from chunk content + * Detect pattern trend from chunk content */ private detectChunkTrend(chunk: CodeChunk): { trend: 'Rising' | 'Stable' | 'Declining' | undefined; @@ -278,6 +316,78 @@ export class CodebaseSearcher { .filter((term) => term.length > 2 && !QUERY_STOP_WORDS.has(term)); } + /** + * Classify query intent based on heuristic patterns + */ + private classifyQueryIntent(query: string): { intent: QueryIntent; weights: IntentWeights } { + const lowerQuery = query.toLowerCase(); + + // EXACT_NAME: Contains PascalCase or camelCase tokens (literal class/component names) + if (/[A-Z][a-z]+[A-Z]/.test(query) || /[a-z][A-Z]/.test(query)) { + return { + intent: 'EXACT_NAME', + weights: { semantic: 0.4, keyword: 0.6 } // Keyword search dominates for exact names + }; + } + + // CONFIG: Configuration/setup queries + const configKeywords = [ + 'config', + 'setup', + 'routing', + 'providers', + 'configuration', + 'bootstrap' + ]; + if (configKeywords.some((kw) => lowerQuery.includes(kw))) { + return { + intent: 'CONFIG', + weights: { semantic: 0.5, keyword: 0.5 } // Balanced + }; + } + + // WIRING: DI/registration queries + const wiringKeywords = [ + 'provide', + 'inject', + 'dependency', + 'register', + 'wire', + 'bootstrap', + 'module' + ]; + if (wiringKeywords.some((kw) => lowerQuery.includes(kw))) { + return { + intent: 'WIRING', + weights: { semantic: 0.5, keyword: 0.5 } // Balanced + }; + } + + // FLOW: Action/navigation queries + const flowVerbs = [ + 'navigate', + 'redirect', + 'route', + 'handle', + 'process', + 'execute', + 'trigger', + 'dispatch' + ]; + if (flowVerbs.some((verb) => lowerQuery.includes(verb))) { + return { + intent: 'FLOW', + weights: { semantic: 0.6, keyword: 0.4 } // Semantic helps with flow understanding + }; + } + + // CONCEPTUAL: Natural language without code tokens (default) + return { + intent: 'CONCEPTUAL', + weights: { semantic: 0.7, keyword: 0.3 } // Semantic dominates for concepts + }; + } + private buildQueryVariants(query: string, maxExpansions: number): QueryVariant[] { const variants: QueryVariant[] = [{ query, weight: 1 }]; if (maxExpansions <= 0) return variants; @@ -312,6 +422,11 @@ export class CodebaseSearcher { return variants.slice(0, 1 + maxExpansions); } + private isTemplateOrStyleFile(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + return ['.html', '.scss', '.css', '.less', '.sass', '.styl'].includes(ext); + } + private isCompositionRootFile(filePath: string): boolean { const normalized = filePath.toLowerCase().replace(/\\/g, '/'); const base = path.basename(normalized); @@ -364,38 +479,96 @@ export class CodebaseSearcher { private scoreAndSortResults( query: string, limit: number, - results: Map, - profile: SearchIntentProfile + results: { + semantic: Map }>; + keyword: Map }>; + }, + profile: SearchIntentProfile, + intent: QueryIntent, + totalVariantWeight: number ): SearchResult[] { const likelyWiringQuery = this.isLikelyWiringOrFlowQuery(query); const actionQuery = this.isActionOrHowQuery(query); - return Array.from(results.entries()) - .map(([_id, { chunk, scores }]) => { - // Calculate base combined score - let combinedScore = scores.reduce((sum, score) => sum + score, 0); + // RRF: k=60 is the standard parameter (proven robust in Elasticsearch + TOSS paper arXiv:2208.11274) + const RRF_K = 60; + + // Collect all unique chunks from both retrieval channels + const allChunks = new Map(); + const rrfScores = new Map(); + + // Gather all chunks + for (const [id, entry] of results.semantic) { + allChunks.set(id, entry.chunk); + } + for (const [id, entry] of results.keyword) { + if (!allChunks.has(id)) { + allChunks.set(id, entry.chunk); + } + } + + // Calculate RRF scores: RRF(d) = SUM(weight_i / (k + rank_i)) + for (const [id] of allChunks) { + let rrfScore = 0; + + // Add contributions from semantic ranks + const semanticEntry = results.semantic.get(id); + if (semanticEntry) { + for (const { rank, weight } of semanticEntry.ranks) { + rrfScore += weight / (RRF_K + rank); + } + } + + // Add contributions from keyword ranks + const keywordEntry = results.keyword.get(id); + if (keywordEntry) { + for (const { rank, weight } of keywordEntry.ranks) { + rrfScore += weight / (RRF_K + rank); + } + } + + rrfScores.set(id, rrfScore); + } + + // Normalize by theoretical maximum (rank-0 in every list), NOT by actual max. + // Using actual max makes top result always 1.0, breaking quality confidence gating. + const theoreticalMaxRrf = totalVariantWeight / (RRF_K + 0); + const maxRrfScore = Math.max(theoreticalMaxRrf, 0.01); + + // Separate test files from implementation files before scoring + const isNonTestQuery = !isTestingRelatedQuery(query); + const implementationChunks: Array<[string, CodeChunk]> = []; + const testChunks: Array<[string, CodeChunk]> = []; + + for (const [id, chunk] of allChunks.entries()) { + if (this.isTestFile(chunk.filePath)) { + testChunks.push([id, chunk]); + } else { + implementationChunks.push([id, chunk]); + } + } + + // For non-test queries: filter test files from candidate pool, keep max 1 test file only if < 3 implementation matches + const chunksToScore = isNonTestQuery ? implementationChunks : Array.from(allChunks.entries()); - // Normalize to 0-1 range (scores are already weighted) - // If both semantic and keyword matched, max possible is ~1.0 - combinedScore = Math.min(1.0, combinedScore); + const scoredResults = chunksToScore + .map(([id, chunk]) => { + // RRF score normalized to [0,1] range. Boosts below are unclamped + // to preserve score differentiation — only relative ordering matters. + let combinedScore = rrfScores.get(id)! / maxRrfScore; // Slight boost when analyzer identified a concrete component type if (chunk.componentType && chunk.componentType !== 'unknown') { - combinedScore = Math.min(1.0, combinedScore * 1.1); + combinedScore *= 1.1; } // Boost if layer is detected if (chunk.layer && chunk.layer !== 'unknown') { - combinedScore = Math.min(1.0, combinedScore * 1.1); - } - - // Query-aware reranking to reduce noisy matches in practical workflows. - if (!isTestingRelatedQuery(query) && this.isTestFile(chunk.filePath)) { - combinedScore = combinedScore * 0.75; + combinedScore *= 1.1; } if (actionQuery && this.isDefinitionHeavyResult(chunk)) { - combinedScore = combinedScore * 0.82; + combinedScore *= 0.82; } if ( @@ -404,27 +577,72 @@ export class CodebaseSearcher { (chunk.componentType || '').toLowerCase() ) ) { - combinedScore = Math.min(1.0, combinedScore * 1.06); + combinedScore *= 1.06; + } + + // Demote template/style files for behavioral queries — they describe + // structure/presentation, not implementation logic. + if (this.isTemplateOrStyleFile(chunk.filePath)) { + combinedScore *= 0.75; } // Light intent-aware boost for likely wiring/configuration queries. if (likelyWiringQuery && profile !== 'explore') { if (this.isCompositionRootFile(chunk.filePath)) { - combinedScore = Math.min(1.0, combinedScore * 1.12); + combinedScore *= 1.12; + } + } + + if (intent === 'FLOW') { + // Boost service/guard/interceptor files for action/navigation queries + if ( + ['service', 'guard', 'interceptor', 'middleware'].includes( + (chunk.componentType || '').toLowerCase() + ) + ) { + combinedScore *= 1.15; + } + } else if (intent === 'CONFIG') { + // Boost composition-root files for configuration queries + if (this.isCompositionRootFile(chunk.filePath)) { + combinedScore *= 1.2; + } + } else if (intent === 'WIRING') { + // Boost DI/module files for wiring queries + if ( + ['module', 'provider', 'config'].some((type) => + (chunk.componentType || '').toLowerCase().includes(type) + ) + ) { + combinedScore *= 1.18; + } + if (this.isCompositionRootFile(chunk.filePath)) { + combinedScore *= 1.22; } } const pathOverlap = this.queryPathTokenOverlap(chunk.filePath, query); if (pathOverlap >= 2) { - combinedScore = Math.min(1.0, combinedScore * 1.08); + combinedScore *= 1.08; } - // v1.2: Detect pattern trend and apply momentum boost + if (this.importCentrality) { + const normalizedRoot = this.rootPath.replace(/\\/g, '/').replace(/\/?$/, '/'); + const normalizedPath = chunk.filePath.replace(/\\/g, '/').replace(normalizedRoot, ''); + const centrality = this.importCentrality.get(normalizedPath); + if (centrality !== undefined && centrality > 0.1) { + // Boost files with high centrality (many imports) + const centralityBoost = 1.0 + centrality * 0.15; // Up to +15% for max centrality + combinedScore *= centralityBoost; + } + } + + // Detect pattern trend and apply momentum boost const { trend, warning } = this.detectChunkTrend(chunk); if (trend === 'Rising') { - combinedScore = Math.min(1.0, combinedScore * 1.15); // +15% for modern patterns + combinedScore *= 1.15; // +15% for modern patterns } else if (trend === 'Declining') { - combinedScore = combinedScore * 0.9; // -10% for legacy patterns + combinedScore *= 0.9; // -10% for legacy patterns } const summary = this.generateSummary(chunk); @@ -443,13 +661,64 @@ export class CodebaseSearcher { componentType: chunk.componentType, layer: chunk.layer, metadata: chunk.metadata, - // v1.2: Pattern momentum awareness trend, patternWarning: warning } as SearchResult; }) - .sort((a, b) => b.score - a.score) - .slice(0, limit); + .sort((a, b) => b.score - a.score); + + const seenFiles = new Set(); + const deduped: SearchResult[] = []; + for (const result of scoredResults) { + const normalizedPath = result.filePath.toLowerCase().replace(/\\/g, '/'); + if (seenFiles.has(normalizedPath)) continue; + seenFiles.add(normalizedPath); + deduped.push(result); + if (deduped.length >= limit) break; + } + const finalResults = deduped; + + if ( + isNonTestQuery && + finalResults.length < 3 && + finalResults.length < limit && + testChunks.length > 0 + ) { + // Find the highest-scoring test file + const bestTestChunk = testChunks + .map(([id, chunk]) => ({ + id, + chunk, + score: rrfScores.get(id)! / maxRrfScore + })) + .sort((a, b) => b.score - a.score)[0]; + + if (bestTestChunk) { + const { trend, warning } = this.detectChunkTrend(bestTestChunk.chunk); + const summary = this.generateSummary(bestTestChunk.chunk); + const snippet = this.generateSnippet(bestTestChunk.chunk.content); + + finalResults.push({ + summary, + snippet, + filePath: bestTestChunk.chunk.filePath, + startLine: bestTestChunk.chunk.startLine, + endLine: bestTestChunk.chunk.endLine, + score: bestTestChunk.score * 0.5, // Demote below implementation files + relevanceReason: + this.generateRelevanceReason(bestTestChunk.chunk, query) + ' (test file)', + language: bestTestChunk.chunk.language, + framework: bestTestChunk.chunk.framework, + componentType: bestTestChunk.chunk.componentType, + layer: bestTestChunk.chunk.layer, + metadata: bestTestChunk.chunk.metadata, + trend, + patternWarning: warning + } as SearchResult); + } + } + + return finalResults; } private pickBetterResultSet( @@ -483,25 +752,38 @@ export class CodebaseSearcher { useKeywordSearch: boolean, semanticWeight: number, keywordWeight: number - ): Promise> { - const results: Map = new Map(); - + ): Promise<{ + semantic: Map }>; + keyword: Map }>; + }> { + const semanticRanks: Map< + string, + { chunk: CodeChunk; ranks: Array<{ rank: number; weight: number }> } + > = new Map(); + const keywordRanks: Map< + string, + { chunk: CodeChunk; ranks: Array<{ rank: number; weight: number }> } + > = new Map(); + + // RRF uses ranks instead of scores for fusion robustness if (useSemanticSearch && this.embeddingProvider && this.storageProvider) { try { for (const variant of queryVariants) { const vectorResults = await this.semanticSearch(variant.query, candidateLimit, filters); - vectorResults.forEach((result) => { + // Assign ranks based on retrieval order (0-indexed) + vectorResults.forEach((result, index) => { const id = result.chunk.id; - const weightedScore = result.score * semanticWeight * variant.weight; - const existing = results.get(id); + const rank = index; // 0-indexed rank + const weight = semanticWeight * variant.weight; + const existing = semanticRanks.get(id); if (existing) { - existing.scores.push(weightedScore); + existing.ranks.push({ rank, weight }); } else { - results.set(id, { + semanticRanks.set(id, { chunk: result.chunk, - scores: [weightedScore] + ranks: [{ rank, weight }] }); } }); @@ -519,17 +801,19 @@ export class CodebaseSearcher { for (const variant of queryVariants) { const keywordResults = await this.keywordSearch(variant.query, candidateLimit, filters); - keywordResults.forEach((result) => { + // Assign ranks based on retrieval order (0-indexed) + keywordResults.forEach((result, index) => { const id = result.chunk.id; - const weightedScore = result.score * keywordWeight * variant.weight; - const existing = results.get(id); + const rank = index; // 0-indexed rank + const weight = keywordWeight * variant.weight; + const existing = keywordRanks.get(id); if (existing) { - existing.scores.push(weightedScore); + existing.ranks.push({ rank, weight }); } else { - results.set(id, { + keywordRanks.set(id, { chunk: result.chunk, - scores: [weightedScore] + ranks: [{ rank, weight }] }); } }); @@ -539,7 +823,7 @@ export class CodebaseSearcher { } } - return results; + return { semantic: semanticRanks, keyword: keywordRanks }; } async search( @@ -552,19 +836,24 @@ export class CodebaseSearcher { await this.initialize(); } + const merged = { + ...DEFAULT_SEARCH_OPTIONS, + ...options + }; const { useSemanticSearch, useKeywordSearch, - semanticWeight, - keywordWeight, profile, enableQueryExpansion, enableLowConfidenceRescue, - candidateFloor - } = { - ...DEFAULT_SEARCH_OPTIONS, - ...options - }; + candidateFloor, + enableReranker + } = merged; + + const { intent, weights: intentWeights } = this.classifyQueryIntent(query); + // Intent weights are the default; caller-supplied weights override them + const finalSemanticWeight = merged.semanticWeight ?? intentWeights.semantic; + const finalKeywordWeight = merged.keywordWeight ?? intentWeights.keyword; const candidateLimit = Math.max(limit * 2, candidateFloor || 30); const primaryVariants = this.buildQueryVariants(query, enableQueryExpansion ? 1 : 0); @@ -575,52 +864,71 @@ export class CodebaseSearcher { filters, Boolean(useSemanticSearch), Boolean(useKeywordSearch), - semanticWeight || 0.7, - keywordWeight || 0.3 + finalSemanticWeight, + finalKeywordWeight ); + const primaryTotalWeight = + primaryVariants.reduce((sum, v) => sum + v.weight, 0) * + (finalSemanticWeight + finalKeywordWeight); const primaryResults = this.scoreAndSortResults( query, limit, primaryMatches, - (profile || 'explore') as SearchIntentProfile + (profile || 'explore') as SearchIntentProfile, + intent, + primaryTotalWeight ); - if (!enableLowConfidenceRescue) { - return primaryResults; - } - - const primaryQuality = assessSearchQuality(query, primaryResults); - if (primaryQuality.status !== 'low_confidence') { - return primaryResults; + let bestResults = primaryResults; + + if (enableLowConfidenceRescue) { + const primaryQuality = assessSearchQuality(query, primaryResults); + if (primaryQuality.status === 'low_confidence') { + const rescueVariants = this.buildQueryVariants(query, 2).slice(1); + if (rescueVariants.length > 0) { + const rescueMatches = await this.collectHybridMatches( + rescueVariants.map((variant, index) => ({ + query: variant.query, + weight: index === 0 ? 1 : 0.8 + })), + candidateLimit, + filters, + Boolean(useSemanticSearch), + Boolean(useKeywordSearch), + finalSemanticWeight, + finalKeywordWeight + ); + + const rescueVariantWeights = rescueVariants.map((_, i) => (i === 0 ? 1 : 0.8)); + const rescueTotalWeight = + rescueVariantWeights.reduce((sum, w) => sum + w, 0) * + (finalSemanticWeight + finalKeywordWeight); + const rescueResults = this.scoreAndSortResults( + query, + limit, + rescueMatches, + (profile || 'explore') as SearchIntentProfile, + intent, + rescueTotalWeight + ); + + bestResults = this.pickBetterResultSet(query, primaryResults, rescueResults); + } + } } - const rescueVariants = this.buildQueryVariants(query, 2).slice(1); - if (rescueVariants.length === 0) { - return primaryResults; + // Stage-2: cross-encoder reranking when top scores are ambiguous + if (enableReranker) { + try { + bestResults = await rerank(query, bestResults); + } catch (error) { + // Reranker is non-critical — log and return unranked results + console.warn('[reranker] Failed, returning original order:', error); + } } - const rescueMatches = await this.collectHybridMatches( - rescueVariants.map((variant, index) => ({ - query: variant.query, - weight: index === 0 ? 1 : 0.8 - })), - candidateLimit, - filters, - Boolean(useSemanticSearch), - Boolean(useKeywordSearch), - semanticWeight || 0.7, - keywordWeight || 0.3 - ); - - const rescueResults = this.scoreAndSortResults( - query, - limit, - rescueMatches, - (profile || 'explore') as SearchIntentProfile - ); - - return this.pickBetterResultSet(query, primaryResults, rescueResults); + return bestResults; } private generateSummary(chunk: CodeChunk): string { diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 64dd6ab..f5621cb 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -1,12 +1,28 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { EmbeddingProvider, DEFAULT_MODEL } from './types.js'; -const MODEL_CONFIGS: Record = { - 'Xenova/bge-small-en-v1.5': { dimensions: 384 }, - 'Xenova/all-MiniLM-L6-v2': { dimensions: 384 }, - 'Xenova/bge-base-en-v1.5': { dimensions: 768 } +interface ModelConfig { + dimensions: number; + maxContext: number; // token context window — used to auto-scale batch size +} + +const MODEL_CONFIGS: Record = { + 'Xenova/bge-small-en-v1.5': { dimensions: 384, maxContext: 512 }, + 'Xenova/all-MiniLM-L6-v2': { dimensions: 384, maxContext: 512 }, + 'Xenova/bge-base-en-v1.5': { dimensions: 768, maxContext: 512 }, + 'onnx-community/granite-embedding-small-english-r2-ONNX': { dimensions: 384, maxContext: 8192 } }; +/** + * Compute a safe batch size for embedding that won't freeze consumer hardware. + * Calibrated so 512-ctx models get batch=32, 8192-ctx models get batch=8. + * Formula: floor(16384 / maxContext), clamped to [4, 32]. + */ +function computeSafeBatchSize(modelName: string): number { + const ctx = MODEL_CONFIGS[modelName]?.maxContext || 512; + return Math.max(4, Math.min(32, Math.floor(16384 / ctx))); +} + export class TransformersEmbeddingProvider implements EmbeddingProvider { readonly name = 'transformers'; readonly modelName: string; @@ -32,12 +48,12 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { private async _initialize(): Promise { try { console.error(`Loading embedding model: ${this.modelName}`); - console.error('(First run will download ~130MB model)'); + console.error('(First run will download the model — this may take a moment)'); - const { pipeline } = await import('@xenova/transformers'); + const { pipeline } = await import('@huggingface/transformers'); this.pipeline = await pipeline('feature-extraction', this.modelName, { - quantized: true + dtype: 'q8' }); this.ready = true; @@ -72,7 +88,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { } const embeddings: number[][] = []; - const batchSize = 32; + const batchSize = computeSafeBatchSize(this.modelName); for (let i = 0; i < texts.length; i += batchSize) { const batch = texts.slice(i, i + batchSize); diff --git a/src/embeddings/types.ts b/src/embeddings/types.ts index add99fc..8a46961 100644 --- a/src/embeddings/types.ts +++ b/src/embeddings/types.ts @@ -19,6 +19,9 @@ export interface EmbeddingConfig { apiEndpoint?: string; } +// Default: bge-small (fast, ~2min indexing, consumer-hardware safe) +// Opt-in: set EMBEDDING_MODEL=onnx-community/granite-embedding-small-english-r2-ONNX for +// better conceptual search at the cost of 5-10x slower indexing and higher RAM usage export const DEFAULT_MODEL = process.env.EMBEDDING_MODEL || 'Xenova/bge-small-en-v1.5'; export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = { diff --git a/src/index.ts b/src/index.ts index 084bc0b..9627e3f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -94,9 +94,7 @@ const PATHS = { vectorDb: path.join(ROOT_PATH, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) }; -// Legacy paths for migration const LEGACY_PATHS = { - // Pre-v1.5 intelligence: path.join(ROOT_PATH, '.codebase-intelligence.json'), keywordIndex: path.join(ROOT_PATH, '.codebase-index.json'), vectorDb: path.join(ROOT_PATH, '.codebase-index') diff --git a/src/types/index.ts b/src/types/index.ts index 634a725..885b296 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -346,14 +346,9 @@ export interface SearchResult { layer?: ArchitecturalLayer; metadata: ChunkMetadata; - // v1.2: Pattern Momentum awareness - /** Pattern trend for this chunk: Rising (modern), Stable, or Declining (legacy) */ trend?: 'Rising' | 'Stable' | 'Declining'; - /** Warning if this result uses declining/legacy patterns */ patternWarning?: string; - // v1.5: Relationship enrichment - /** Structured relationship data from the import graph and git history */ relationships?: RelationshipData; // Optional detailed context (for agent to request if needed) diff --git a/src/utils/chunking.ts b/src/utils/chunking.ts index 334fc28..2d47dfa 100644 --- a/src/utils/chunking.ts +++ b/src/utils/chunking.ts @@ -14,8 +14,8 @@ export interface ChunkingOptions { } const DEFAULT_OPTIONS: ChunkingOptions = { - maxChunkSize: 100, - overlapSize: 10, + maxChunkSize: 50, + overlapSize: 0, preserveBoundaries: true }; diff --git a/src/utils/usage-tracker.ts b/src/utils/usage-tracker.ts index 01d8c88..32144c4 100644 --- a/src/utils/usage-tracker.ts +++ b/src/utils/usage-tracker.ts @@ -432,11 +432,9 @@ export class PatternDetector { const exampleKey = `${category}:${primaryName}`; const canonicalExample = this.canonicalExamples.get(exampleKey); - // v1.2 Fix: Use P90 robust date instead of raw max date const primaryDate = this.getRobustDate(category, primaryName); const primaryTrend = calculateTrend(primaryDate); - // v1.2 Fix: Detect if any alternative is Rising to inform primary guidance let hasRisingAlternative = false; let alternatives: Array<{ name: string; diff --git a/tests/eval-harness.test.ts b/tests/eval-harness.test.ts new file mode 100644 index 0000000..33c3795 --- /dev/null +++ b/tests/eval-harness.test.ts @@ -0,0 +1,381 @@ +/** + * Evaluation Harness for Search Quality + * + * Tests search quality against ground-truth queries defined in + * tests/fixtures/eval-angular-spotify.json. Uses pattern-based matching + * against a public codebase for reproducible evaluation. + * + * Metrics: + * - Top-1 accuracy: Is the correct file the top result? (unique files) + * - Top-3 recall: Is the correct file in the top 3 unique files? + * - Spec contamination: Are test/spec files incorrectly dominating results? + * + * Gate: 14/20 correct top-1 results (70% threshold) + * + * Usage: This test is designed to work with MOCKED search results for unit testing. + * For live evaluation against a real index, use the `evaluateSearchQuality()` function + * directly with a CodebaseSearcher instance. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { CodeChunk, SearchResult } from '../src/types/index.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import evalFixture from './fixtures/eval-angular-spotify.json'; + +// ─── Types ─────────────────────────────────────────────────────────────────── + +interface EvalQuery { + id: number; + query: string; + category: string; + expectedPatterns: string[]; + expectedNotPatterns: string[]; + notes: string; +} + +interface EvalResult { + queryId: number; + query: string; + category: string; + topFile: string | null; + top3Files: string[]; + top1Correct: boolean; + top3Recall: boolean; + specContaminated: boolean; + score: number; +} + +interface EvalSummary { + total: number; + top1Correct: number; + top1Accuracy: number; + top3Recall: number; + specContaminationRate: number; + avgTopScore: number; + results: EvalResult[]; + passesGate: boolean; +} + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +function isTestFile(filePath: string): boolean { + const normalized = filePath.toLowerCase().replace(/\\/g, '/'); + return ( + normalized.includes('.spec.') || + normalized.includes('.test.') || + normalized.includes('/e2e/') || + normalized.includes('/__tests__/') + ); +} + +function matchesExpected(filePath: string, expectedPatterns: string[]): boolean { + const normalized = filePath.toLowerCase().replace(/\\/g, '/'); + return expectedPatterns.some((pattern) => normalized.includes(pattern.toLowerCase())); +} + +function matchesNotExpected(filePath: string, notExpectedPatterns: string[]): boolean { + const normalized = filePath.toLowerCase().replace(/\\/g, '/'); + return notExpectedPatterns.some((pattern) => normalized.includes(pattern.toLowerCase())); +} + +/** + * Deduplicate results to unique file paths (keep best-scored chunk per file). + * Search already dedupes, but this makes the harness robust if called directly. + */ +function dedupeByFile(results: SearchResult[]): SearchResult[] { + const seen = new Set(); + const deduped: SearchResult[] = []; + for (const r of results) { + const key = r.filePath.toLowerCase().replace(/\\/g, '/'); + if (seen.has(key)) continue; + seen.add(key); + deduped.push(r); + } + return deduped; +} + +/** + * Evaluate a single query result against ground truth. + * Results are deduplicated to unique files before scoring. + */ +function evaluateQuery(query: EvalQuery, results: SearchResult[]): EvalResult { + const uniqueResults = dedupeByFile(results); + const topFile = uniqueResults.length > 0 ? uniqueResults[0].filePath : null; + const top3Files = uniqueResults.slice(0, 3).map((r) => r.filePath); + + const top1Correct = topFile !== null && matchesExpected(topFile, query.expectedPatterns) && + !matchesNotExpected(topFile, query.expectedNotPatterns); + + const top3Recall = top3Files.some( + (f) => matchesExpected(f, query.expectedPatterns) && !matchesNotExpected(f, query.expectedNotPatterns) + ); + + const specCount = top3Files.filter((f) => isTestFile(f)).length; + const specContaminated = specCount >= 2; // 2+ spec files in top 3 = contaminated + + return { + queryId: query.id, + query: query.query, + category: query.category, + topFile, + top3Files, + top1Correct, + top3Recall, + specContaminated, + score: uniqueResults.length > 0 ? uniqueResults[0].score : 0 + }; +} + +/** + * Run full evaluation and return summary. + * This is the main function that both agents use to measure delta. + */ +export function summarizeEvaluation(results: EvalResult[]): EvalSummary { + const total = results.length; + const top1Correct = results.filter((r) => r.top1Correct).length; + const top3RecallCount = results.filter((r) => r.top3Recall).length; + const specContaminated = results.filter((r) => r.specContaminated).length; + const avgTopScore = + results.length > 0 ? results.reduce((sum, r) => sum + r.score, 0) / results.length : 0; + + return { + total, + top1Correct, + top1Accuracy: top1Correct / total, + top3Recall: top3RecallCount / total, + specContaminationRate: specContaminated / total, + avgTopScore, + results, + passesGate: top1Correct >= 14 // 14/20 = 70% gate + }; +} + +/** + * Run evaluation against a live CodebaseSearcher instance. + * This is the function to use for real measurement runs. + */ +export async function evaluateSearchQuality( + searcher: CodebaseSearcher, + limit: number = 5 +): Promise { + const queries = evalFixture.queries as EvalQuery[]; + const results: EvalResult[] = []; + + for (const query of queries) { + const searchResults = await searcher.search(query.query, limit); + results.push(evaluateQuery(query, searchResults)); + } + + return summarizeEvaluation(results); +} + +/** + * Pretty-print evaluation summary for console output + */ +export function printEvalSummary(summary: EvalSummary): string { + const lines: string[] = [ + `\n=== Search Quality Evaluation ===`, + `Top-1 Accuracy: ${summary.top1Correct}/${summary.total} (${(summary.top1Accuracy * 100).toFixed(0)}%)`, + `Top-3 Recall: ${(summary.top3Recall * 100).toFixed(0)}%`, + `Spec Contamination: ${(summary.specContaminationRate * 100).toFixed(0)}%`, + `Avg Top Score: ${summary.avgTopScore.toFixed(3)}`, + `Gate (14/20): ${summary.passesGate ? 'PASS' : 'FAIL'}`, + ``, + `Per-query breakdown:` + ]; + + for (const r of summary.results) { + const status = r.top1Correct ? 'PASS' : 'FAIL'; + const specNote = r.specContaminated ? ' [SPEC CONTAMINATED]' : ''; + const topFileShort = r.topFile ? r.topFile.split(/[\\/]/).pop() : 'none'; + lines.push( + ` ${status} [${r.category}] "${r.query}" -> ${topFileShort} (${r.score.toFixed(3)})${specNote}` + ); + } + + lines.push(`\n================================\n`); + return lines.join('\n'); +} + +// ─── Unit Tests (mocked) ───────────────────────────────────────────────────── + +function createChunk( + id: string, + filePath: string, + content: string, + overrides?: Partial +): CodeChunk { + return { + id, + content, + filePath, + relativePath: filePath.replace(/^.*?[/\\](?:src|libs|apps)[/\\]/, ''), + startLine: 1, + endLine: 40, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {}, + ...overrides + }; +} + +function setupSearcherWithResults( + results: { chunk: CodeChunk; score: number }[] +): CodebaseSearcher { + const searcher = new CodebaseSearcher('C:/repo') as any; + searcher.initialized = true; + searcher.embeddingProvider = { + embed: vi.fn(async () => new Array(384).fill(0.01)) + }; + searcher.storageProvider = { + search: vi.fn(async () => results), + count: vi.fn(async () => results.length) + }; + searcher.fuseIndex = null; + searcher.patternIntelligence = null; + return searcher as CodebaseSearcher; +} + +describe('Eval Harness - fixtures loaded', () => { + it('has 20 evaluation queries', () => { + expect(evalFixture.queries).toHaveLength(20); + }); + + it('covers all four categories', () => { + const categories = new Set(evalFixture.queries.map((q: EvalQuery) => q.category)); + expect(categories.has('exact-name')).toBe(true); + expect(categories.has('conceptual')).toBe(true); + expect(categories.has('multi-concept')).toBe(true); + expect(categories.has('structural')).toBe(true); + }); + + it('each query has expectedPatterns', () => { + for (const q of evalFixture.queries as EvalQuery[]) { + expect(q.expectedPatterns.length).toBeGreaterThan(0); + } + }); +}); + +describe('Eval Harness - scoring logic', () => { + it('marks correct top-1 when implementation file is first', () => { + const query: EvalQuery = { + id: 7, + query: 'authentication login', + category: 'conceptual', + expectedPatterns: ['auth.service'], + expectedNotPatterns: ['.spec.', '/e2e/'], + notes: '' + }; + const results: SearchResult[] = [ + { filePath: 'src/services/auth/auth.service.ts', score: 0.65 } as SearchResult, + { filePath: 'src/e2e/setup.ts', score: 0.55 } as SearchResult + ]; + + const evalResult = evaluateQuery(query, results); + expect(evalResult.top1Correct).toBe(true); + expect(evalResult.specContaminated).toBe(false); + }); + + it('marks FAIL when spec file is top-1 for non-test query', () => { + const query: EvalQuery = { + id: 4, + query: 'add authorization token to API requests', + category: 'conceptual', + expectedPatterns: ['auth', 'interceptor'], + expectedNotPatterns: ['.spec.', '.test.'], + notes: '' + }; + const results: SearchResult[] = [ + { filePath: 'src/interceptors/auth.interceptor.spec.ts', score: 0.45 } as SearchResult, + { filePath: 'src/interceptors/error.interceptor.spec.ts', score: 0.42 } as SearchResult, + { filePath: 'src/services/api.service.spec.ts', score: 0.39 } as SearchResult + ]; + + const evalResult = evaluateQuery(query, results); + expect(evalResult.top1Correct).toBe(false); + expect(evalResult.specContaminated).toBe(true); + }); + + it('detects spec contamination when 2+ spec files in top 3', () => { + const query: EvalQuery = { + id: 3, + query: 'persist data across browser sessions', + category: 'conceptual', + expectedPatterns: ['storage'], + expectedNotPatterns: ['.spec.'], + notes: '' + }; + const results: SearchResult[] = [ + { filePath: 'src/services/local-storage.service.ts', score: 0.5 } as SearchResult, + { filePath: 'src/services/local-storage.service.spec.ts', score: 0.48 } as SearchResult, + { filePath: 'src/services/session.service.spec.ts', score: 0.45 } as SearchResult + ]; + + const evalResult = evaluateQuery(query, results); + expect(evalResult.top1Correct).toBe(true); // implementation is top-1 + expect(evalResult.specContaminated).toBe(true); // but 2/3 are specs + }); + + it('summarizeEvaluation calculates gate correctly', () => { + const passing: EvalResult[] = Array(14).fill(null).map((_, i) => ({ + queryId: i + 1, query: `q${i}`, category: 'conceptual', + topFile: 'correct.ts', top3Files: ['correct.ts'], top1Correct: true, + top3Recall: true, specContaminated: false, score: 0.6 + })); + const failing: EvalResult[] = Array(6).fill(null).map((_, i) => ({ + queryId: i + 15, query: `q${i + 14}`, category: 'multi-concept', + topFile: 'wrong.spec.ts', top3Files: ['wrong.spec.ts'], top1Correct: false, + top3Recall: false, specContaminated: true, score: 0.3 + })); + + const summary = summarizeEvaluation([...passing, ...failing]); + expect(summary.total).toBe(20); + expect(summary.top1Correct).toBe(14); + expect(summary.passesGate).toBe(true); + expect(summary.top1Accuracy).toBeCloseTo(14 / 20, 2); + }); + + it('fails gate when only 13/20 pass', () => { + const results: EvalResult[] = Array(20).fill(null).map((_, i) => ({ + queryId: i + 1, query: `q${i}`, category: 'conceptual', + topFile: i < 13 ? 'correct.ts' : 'wrong.ts', top3Files: [], + top1Correct: i < 13, top3Recall: i < 13, specContaminated: false, score: 0.5 + })); + + const summary = summarizeEvaluation(results); + expect(summary.top1Correct).toBe(13); + expect(summary.passesGate).toBe(false); + }); +}); + +describe('Eval Harness - integration with CodebaseSearcher (mocked)', () => { + it('runs the auth service query correctly through searcher', async () => { + const authServiceChunk = createChunk( + 'auth-svc', + 'C:/repo/src/services/auth/auth.service.ts', + 'export class AuthService { login() {} isLoggedIn$ = new BehaviorSubject(false); }', + { componentType: 'service', metadata: { componentName: 'AuthService' } } + ); + const e2eChunk = createChunk( + 'e2e-auth', + 'C:/repo/src/e2e/setup.ts', + 'export async function authenticate() { await page.fill(username); }', + { componentType: 'unknown' } + ); + + const searcher = setupSearcherWithResults([ + { chunk: authServiceChunk, score: 0.65 }, + { chunk: e2eChunk, score: 0.55 } + ]); + + const results = await searcher.search('AuthService', 5); + expect(results.length).toBeGreaterThan(0); + expect(results[0].filePath).toContain('auth.service.ts'); + }); +}); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..36abf2f --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,138 @@ +# Evaluation Fixtures + +This directory contains frozen evaluation sets for testing code search quality. + +## Files + +- `eval-angular-spotify.json` - 20 semantic queries against [angular-spotify](https://github.com/trungk18/angular-spotify) (public, reproducible) + +## Running Evaluations + +### Prerequisites + +1. Clone the test codebase: +```bash +git clone https://github.com/trungk18/angular-spotify /path/to/angular-spotify +``` + +2. Build this project: +```bash +npm install +npm run build +``` + +### Run Evaluation + +```bash +node scripts/run-eval.mjs /path/to/angular-spotify --fixture tests/fixtures/eval-angular-spotify.json +``` + +### Output Format + +The eval script outputs: +- **Top-1 Accuracy**: % of queries where the best result matches expected patterns +- **Top-3 Recall**: % of queries where top-3 results include a match +- **Spec Contamination**: % of queries returning test files +- **Per-category breakdown**: Accuracy by query type (exact-name, conceptual, multi-concept, structural) +- **Failure analysis**: Which queries failed and why + +## Evaluation Integrity Rules + +⚠️ **CRITICAL**: These eval fixtures are FROZEN. Once committed: + +1. **DO NOT** adjust expected results to match system output +2. **DO NOT** add queries during development to "improve" scores +3. **DO NOT** remove "hard" queries that the system fails +4. **DO NOT** tune the system on this eval set then report scores + +### Proper Usage + +✅ **CORRECT**: +- Commit frozen eval BEFORE making changes +- Use eval to measure improvement honestly +- Report failures transparently +- Create NEW eval sets for iteration + +❌ **INCORRECT**: +- Adjusting fixture during development ("fixture fixes") +- Cherry-picking queries that work well +- Overfitting to this specific codebase +- Reporting scores without disclosing methodology + +## Query Design Principles + +### Semantic Queries (NOT keyword matching) + +Queries are designed to test **semantic understanding**, not keyword matching: + +- ✅ "skip to next song" → should find `player-api.ts` (no "skip" keyword in file) +- ✅ "persist data across browser sessions" → should find `local-storage.service.ts` +- ✅ "add authorization token to API requests" → should find `auth.interceptor.ts` + +- ❌ "PlayerApiService" → keyword match (too easy) +- ❌ "player api" → keyword match (too easy) + +### Expected Patterns (NOT specific paths) + +Expected results use **patterns** that work across codebases: + +```json +{ + "expectedPatterns": ["player", "api"], + "expectedNotPatterns": [".spec.", ".test."] +} +``` + +This matches: +- `libs/web/shared/data-access/spotify-api/src/lib/player-api.ts` ✅ +- `apps/music/src/services/player-api.service.ts` ✅ +- `player-api.spec.ts` ❌ (excluded by expectedNotPatterns) + +### Query Categories + +1. **conceptual** (7 queries): Natural language descriptions requiring semantic understanding +2. **multi-concept** (7 queries): Combining multiple concepts (hardest) +3. **exact-name** (3 queries): Class/service names (baseline) +4. **structural** (3 queries): Framework-specific patterns (NgRx, interceptors) + +## Ground Truth Verification + +Ground truth established via manual code review: + +1. Read the actual code to understand what it does +2. Verify the expected file implements the described functionality +3. Check for similar files that should also match +4. Document reasoning in query notes + +Example: +- Query: "skip to next song" +- Expected: `player-api.ts` +- Reasoning: File contains `next()` method that calls `/me/player/next` API endpoint + +## Reproducing Results + +To reproduce published results: + +1. Clone the exact codebase version: +```bash +git clone https://github.com/trungk18/angular-spotify +cd angular-spotify +git checkout +``` + +2. Use the frozen eval fixture (committed before measurements) +3. Run eval on both baseline and new version +4. Compare metrics transparently + +## Adding New Eval Sets + +When creating new eval sets: + +1. Design queries BEFORE any implementation +2. Establish ground truth via manual review +3. Test on multiple codebases (not just one) +4. Include "hard" queries expected to fail +5. Commit and tag BEFORE running any measurements +6. Document methodology in query notes + +See this README for full guidelines. diff --git a/tests/fixtures/eval-angular-spotify.json b/tests/fixtures/eval-angular-spotify.json new file mode 100644 index 0000000..fdf9312 --- /dev/null +++ b/tests/fixtures/eval-angular-spotify.json @@ -0,0 +1,169 @@ +{ + "description": "Generic eval set for Angular code search quality - 20 semantic queries", + "codebase": "angular-spotify", + "repository": "https://github.com/trungk18/angular-spotify", + "frozenDate": "2026-02-09", + "notes": "Queries designed to test SEMANTIC understanding, not keyword matching. Expected patterns are file name patterns that should match, not full paths.", + "queries": [ + { + "id": 1, + "query": "skip to next song", + "category": "conceptual", + "expectedPatterns": ["player"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find player API with next() method. Tests understanding 'skip' -> 'next'" + }, + { + "id": 2, + "query": "adjust audio volume", + "category": "conceptual", + "expectedPatterns": ["player"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find player API with setVolume() method. Tests 'adjust volume' concept" + }, + { + "id": 3, + "query": "persist data across browser sessions", + "category": "conceptual", + "expectedPatterns": ["storage"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find local-storage service. Tests understanding persistence concept" + }, + { + "id": 4, + "query": "add authorization token to API requests", + "category": "conceptual", + "expectedPatterns": ["auth", "interceptor"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find auth interceptor. Tests understanding of auth headers" + }, + { + "id": 5, + "query": "handle side effects from user actions", + "category": "conceptual", + "expectedPatterns": ["effect"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find NgRx effects. Tests understanding of side effect pattern" + }, + { + "id": 6, + "query": "switch playback to different device", + "category": "conceptual", + "expectedPatterns": ["player"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find player API with transferUserPlayback(). Tests device switching concept" + }, + { + "id": 7, + "query": "remember user settings after page reload", + "category": "conceptual", + "expectedPatterns": ["storage", "settings"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find local-storage or settings services. Tests persistence concept" + }, + { + "id": 8, + "query": "fetch recently played music", + "category": "conceptual", + "expectedPatterns": ["player", "recent"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find player API with getRecentPlayedTracks(). Tests history concept" + }, + { + "id": 9, + "query": "track user interactions for analytics", + "category": "conceptual", + "expectedPatterns": ["analytics", "google"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find google-analytics service. Tests tracking concept" + }, + { + "id": 10, + "query": "retrieve user's saved music collection", + "category": "conceptual", + "expectedPatterns": ["album", "api"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find album API with getUserSavedAlbums(). Tests library concept" + }, + { + "id": 11, + "query": "handle unauthorized API responses", + "category": "multi-concept", + "expectedPatterns": ["unauthorized", "interceptor"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find unauthorized interceptor. Tests error handling + auth" + }, + { + "id": 12, + "query": "manage application state for music albums", + "category": "multi-concept", + "expectedPatterns": ["album", "store"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find album store/reducer. Tests state management + domain" + }, + { + "id": 13, + "query": "dispatch actions to update UI state", + "category": "multi-concept", + "expectedPatterns": ["action"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find *.action.ts files. Tests NgRx action concept" + }, + { + "id": 14, + "query": "browse music categories from API", + "category": "multi-concept", + "expectedPatterns": ["browse", "categor"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find browse/categories API or store. Tests discovery + data fetching" + }, + { + "id": 15, + "query": "select data from application store", + "category": "multi-concept", + "expectedPatterns": ["selector"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Should find *.selector.ts files. Tests NgRx selector concept" + }, + { + "id": 16, + "query": "AlbumApiService", + "category": "exact-name", + "expectedPatterns": ["album", "api"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Exact class name lookup. Should find album-api.ts" + }, + { + "id": 17, + "query": "PlaylistApiService", + "category": "exact-name", + "expectedPatterns": ["playlist", "api"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Exact class name lookup. Should find playlist-api.ts" + }, + { + "id": 18, + "query": "LocalStorageService", + "category": "exact-name", + "expectedPatterns": ["storage"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Exact class name lookup. Should find local-storage.service.ts" + }, + { + "id": 19, + "query": "ngrx effects implementation", + "category": "structural", + "expectedPatterns": ["effect"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Framework-specific pattern. Should find *.effect.ts files" + }, + { + "id": 20, + "query": "http interceptor configuration", + "category": "structural", + "expectedPatterns": ["interceptor"], + "expectedNotPatterns": [".spec.", ".test."], + "notes": "Framework-specific pattern. Should find *.interceptor.ts files" + } + ] +} diff --git a/tests/reranker.test.ts b/tests/reranker.test.ts new file mode 100644 index 0000000..7bee16c --- /dev/null +++ b/tests/reranker.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect } from 'vitest'; +import { isAmbiguous } from '../src/core/reranker.js'; +import type { SearchResult } from '../src/types/index.js'; + +function makeResult(score: number, filePath: string): SearchResult { + return { + summary: `Result from ${filePath}`, + snippet: 'export class Foo {}', + filePath, + startLine: 1, + endLine: 10, + score, + language: 'typescript', + metadata: {} + } as SearchResult; +} + +describe('Reranker ambiguity detection', () => { + it('detects ambiguous results when top scores are clustered', () => { + const results = [ + makeResult(0.85, '/a.ts'), + makeResult(0.83, '/b.ts'), + makeResult(0.82, '/c.ts') + ]; + expect(isAmbiguous(results)).toBe(true); + }); + + it('returns false when there is a clear winner', () => { + const results = [ + makeResult(0.95, '/a.ts'), + makeResult(0.75, '/b.ts'), + makeResult(0.60, '/c.ts') + ]; + expect(isAmbiguous(results)).toBe(false); + }); + + it('returns false for fewer than 3 results', () => { + const results = [ + makeResult(0.85, '/a.ts'), + makeResult(0.84, '/b.ts') + ]; + expect(isAmbiguous(results)).toBe(false); + }); + + it('correctly handles edge case at threshold boundary', () => { + // Gap of exactly 0.08 is NOT below the < 0.08 threshold + const results = [ + makeResult(0.90, '/a.ts'), + makeResult(0.85, '/b.ts'), + makeResult(0.82, '/c.ts') + ]; + expect(isAmbiguous(results)).toBe(false); // 0.90 - 0.82 = 0.08, not < threshold + + // Gap of 0.07 IS below the threshold + const ambiguous = [ + makeResult(0.90, '/a.ts'), + makeResult(0.86, '/b.ts'), + makeResult(0.83, '/c.ts') + ]; + expect(isAmbiguous(ambiguous)).toBe(true); // 0.90 - 0.83 = 0.07, below threshold + }); +}); + +describe('File-level dedupe in search results', () => { + it('removes duplicate files keeping best score via CodebaseSearcher', async () => { + // This is tested via the integration in search-ranking.test.ts + // The dedupe logic is in scoreAndSortResults — tested indirectly by + // ensuring results contain unique file paths + const { CodebaseSearcher } = await import('../src/core/search.js'); + const { vi } = await import('vitest'); + + const searcher = new CodebaseSearcher('C:/repo') as any; + searcher.initialized = true; + searcher.embeddingProvider = { + embed: vi.fn(async () => [0.1, 0.2]) + }; + searcher.storageProvider = { + search: vi.fn(async () => [ + { + chunk: { + id: 'chunk1', + content: 'class Foo {}', + filePath: 'C:/repo/src/foo.ts', + relativePath: 'src/foo.ts', + startLine: 1, + endLine: 10, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + }, + score: 0.9 + }, + { + chunk: { + id: 'chunk2', + content: 'class Bar extends Foo {}', + filePath: 'C:/repo/src/foo.ts', // same file, different chunk + relativePath: 'src/foo.ts', + startLine: 11, + endLine: 20, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + }, + score: 0.85 + }, + { + chunk: { + id: 'chunk3', + content: 'class Baz {}', + filePath: 'C:/repo/src/baz.ts', // different file + relativePath: 'src/baz.ts', + startLine: 1, + endLine: 10, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + }, + score: 0.80 + } + ]), + count: vi.fn(async () => 3) + }; + searcher.fuseIndex = null; + searcher.patternIntelligence = null; + + const results = await (searcher as any).search('Foo class', 5, undefined, { + useSemanticSearch: true, + useKeywordSearch: false, + enableReranker: false, + enableLowConfidenceRescue: false, + enableQueryExpansion: false + }); + + // Should have 2 unique files, not 3 results (two from foo.ts) + const filePaths = results.map((r: any) => r.filePath); + expect(filePaths.length).toBe(2); + expect(filePaths[0]).toContain('foo.ts'); + expect(filePaths[1]).toContain('baz.ts'); + }); +}); diff --git a/tests/search-retrieval-strategy.test.ts b/tests/search-retrieval-strategy.test.ts index 065f939..0cb92a2 100644 --- a/tests/search-retrieval-strategy.test.ts +++ b/tests/search-retrieval-strategy.test.ts @@ -118,4 +118,107 @@ describe('CodebaseSearcher retrieval strategy', () => { expect(results[0].filePath).toContain('auth-callback.component.ts'); expect(results[0].filePath).not.toContain('.spec.ts'); }); + + it('applies intent-based keyword-heavy weights for exact-name queries', async () => { + const implChunk = createChunk( + 'impl', + 'src/core/auth-service.ts', + 'export class AuthService { login() {} }' + ); + + const searcher = new CodebaseSearcher('C:/repo') as any; + searcher.initialized = true; + searcher.embeddingProvider = {}; + searcher.storageProvider = {}; + searcher.fuseIndex = null; + searcher.patternIntelligence = null; + + let capturedSemanticWeight: number | undefined; + let capturedKeywordWeight: number | undefined; + + // Intercept collectHybridMatches to capture the weights used + searcher.collectHybridMatches = vi.fn( + async ( + _variants: any, + _limit: any, + _filters: any, + _useSemantic: any, + _useKeyword: any, + semWeight: number, + kwWeight: number + ) => { + capturedSemanticWeight = semWeight; + capturedKeywordWeight = kwWeight; + return { + semantic: new Map([ + ['impl', { chunk: implChunk, ranks: [{ rank: 0, weight: semWeight }] }] + ]), + keyword: new Map([ + ['impl', { chunk: implChunk, ranks: [{ rank: 0, weight: kwWeight }] }] + ]) + }; + } + ); + + // AuthService is PascalCase → EXACT_NAME intent → keyword: 0.6, semantic: 0.4 + await searcher.search('AuthService', 3, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: false + }); + + expect(capturedKeywordWeight).toBeGreaterThan(capturedSemanticWeight!); + expect(capturedKeywordWeight).toBeCloseTo(0.6, 1); + expect(capturedSemanticWeight).toBeCloseTo(0.4, 1); + }); + + it('produces scores below 1.0 for weak single-list retrievals', async () => { + const strongChunk = createChunk( + 'strong', + 'src/core/auth-service.ts', + 'export class AuthService { login() {} }' + ); + const weakChunk = createChunk('weak', 'src/utils/helpers.ts', 'export function helper() {}'); + + const searcher = new CodebaseSearcher('C:/repo') as any; + searcher.initialized = true; + searcher.embeddingProvider = {}; + searcher.storageProvider = {}; + searcher.fuseIndex = null; + searcher.patternIntelligence = null; + + // Mock collectHybridMatches: strong chunk rank 0 in both channels, + // weak chunk rank 5 in keyword only + searcher.collectHybridMatches = vi.fn( + async ( + _variants: any, + _limit: any, + _filters: any, + _useSemantic: any, + _useKeyword: any, + semWeight: number, + kwWeight: number + ) => ({ + semantic: new Map([ + ['strong', { chunk: strongChunk, ranks: [{ rank: 0, weight: semWeight }] }] + ]), + keyword: new Map([ + ['strong', { chunk: strongChunk, ranks: [{ rank: 0, weight: kwWeight }] }], + ['weak', { chunk: weakChunk, ranks: [{ rank: 5, weight: kwWeight }] }] + ]) + }) + ); + + const results = await searcher.search('authentication login', 10, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: false + }); + + expect(results.length).toBeGreaterThanOrEqual(2); + expect(results[0].score).toBeGreaterThan(0); + // Key invariant: strong result (rank 0, both channels) must score + // meaningfully higher than weak result (rank 5, keyword only). + // With normalization-by-actual-max, the ratio collapses. + const ratio = results[0].score / results[1].score; + expect(ratio).toBeGreaterThan(1.5); + }); });