From 04cd6569474288c77bb6f4f205874a2221ec2137 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Wed, 23 Sep 2026 23:55:12 -0400 Subject: [PATCH 1/2] fix(web): deduplicate repository lookups within search chunks --- .../src/features/search/zoektSearcher.test.ts | 94 ++++++++++++++++++- .../web/src/features/search/zoektSearcher.ts | 5 +- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/packages/web/src/features/search/zoektSearcher.test.ts b/packages/web/src/features/search/zoektSearcher.test.ts index 0bdb72818..41c5f4631 100644 --- a/packages/web/src/features/search/zoektSearcher.test.ts +++ b/packages/web/src/features/search/zoektSearcher.test.ts @@ -1,13 +1,16 @@ import type { PrismaClient } from '@sourcebot/db'; import type { SearchRequest as ZoektGrpcSearchRequest } from '@/proto/zoekt/webserver/v1/SearchRequest'; import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; const mocks = vi.hoisted(() => { const close = vi.fn(); const search = vi.fn(); + const streamSearch = vi.fn(); class WebserverService { Search = search; + StreamSearch = streamSearch; close = close; } @@ -15,6 +18,7 @@ const mocks = vi.hoisted(() => { close, loadSync: vi.fn(() => ({})), search, + streamSearch, WebserverService, }; }); @@ -60,10 +64,27 @@ vi.mock('@/lib/posthog', () => ({ captureEvent: vi.fn(), })); -import { zoektSearch } from './zoektSearcher'; +import { zoektSearch, zoektStreamSearch } from './zoektSearcher'; const searchRequest = {} as ZoektGrpcSearchRequest; +const createFile = (id: number | undefined, repository = 'github.com/org/repo') => ({ + repository_id: id, + repository, + file_name: Buffer.from('src/index.ts'), + chunk_matches: [], + branches: ['main'], + language: 'TypeScript', +}); + +const createRepo = (id: number, name = 'github.com/org/repo') => ({ + id, + name, + displayName: name, + webUrl: null, + external_codeHostType: 'github', +}); + describe('zoektSearch', () => { beforeEach(() => { vi.clearAllMocks(); @@ -104,4 +125,75 @@ describe('zoektSearch', () => { await expect(zoektSearch(searchRequest, prisma)).rejects.toThrow('database unavailable'); expect(mocks.close).toHaveBeenCalledOnce(); }); + + test.each([1, 2])('looks up each of %i repositories only once for 100 files', async (repoCount) => { + const files = Array.from({ length: 100 }, (_, index) => createFile(index % repoCount + 1)); + mocks.search.mockImplementation((_request, _metadata, callback) => { + callback(null, { files }); + }); + const findUnique = vi.fn(async ({ where: { id } }) => createRepo(id)); + const prisma = { repo: { findUnique } } as unknown as PrismaClient; + + const response = await zoektSearch(searchRequest, prisma); + + expect(findUnique).toHaveBeenCalledTimes(repoCount); + expect(response.files).toHaveLength(100); + expect(response.files.map(file => file.repositoryId)).toEqual(files.map(file => file.repository_id)); + expect(response.repositoryInfo).toHaveLength(repoCount); + }); + + test('deduplicates lookups by name for legacy shards without repository IDs', async () => { + mocks.search.mockImplementation((_request, _metadata, callback) => { + callback(null, { files: Array.from({ length: 100 }, () => createFile(undefined)) }); + }); + const findFirst = vi.fn().mockResolvedValue(createRepo(1)); + const prisma = { repo: { findFirst } } as unknown as PrismaClient; + + const response = await zoektSearch(searchRequest, prisma); + + expect(findFirst).toHaveBeenCalledExactlyOnceWith({ where: { name: 'github.com/org/repo' } }); + expect(response.files).toHaveLength(100); + }); + + test('looks up a missing repository once and omits its files', async () => { + mocks.search.mockImplementation((_request, _metadata, callback) => { + callback(null, { files: Array.from({ length: 100 }, () => createFile(1)) }); + }); + const findUnique = vi.fn().mockResolvedValue(null); + const prisma = { repo: { findUnique } } as unknown as PrismaClient; + + const response = await zoektSearch(searchRequest, prisma); + + expect(findUnique).toHaveBeenCalledOnce(); + expect(response.files).toEqual([]); + expect(response.repositoryInfo).toEqual([]); + }); + + test('deduplicates streaming chunks and reuses repository metadata across chunks', async () => { + const grpcStream = Object.assign(new EventEmitter(), { + pause: vi.fn(), + resume: vi.fn(), + cancel: vi.fn(), + }); + mocks.streamSearch.mockReturnValue(grpcStream); + const findUnique = vi.fn(async ({ where: { id } }) => createRepo(id)); + const prisma = { repo: { findUnique } } as unknown as PrismaClient; + const stream = await zoektStreamSearch(searchRequest, prisma); + const reader = stream.getReader(); + + for (const ids of [[1, 1, 2, 2], [1, 2, 3, 3]]) { + grpcStream.emit('data', { response_chunk: { files: ids.map(id => createFile(id)) } }); + const chunk = await reader.read(); + const response = JSON.parse(new TextDecoder().decode(chunk.value).slice('data: '.length)); + expect(response.files).toHaveLength(ids.length); + expect(response.repositoryInfo).toHaveLength(new Set(ids).size); + } + + expect(findUnique).toHaveBeenCalledTimes(3); + grpcStream.emit('end'); + while (!(await reader.read()).done) { + // Drain the final statistics and completion marker. + } + expect(mocks.close).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/web/src/features/search/zoektSearcher.ts b/packages/web/src/features/search/zoektSearcher.ts index 087e44043..c19aa222e 100644 --- a/packages/web/src/features/search/zoektSearcher.ts +++ b/packages/web/src/features/search/zoektSearcher.ts @@ -341,9 +341,8 @@ const encodeSSEREsponseChunk = (response: object | string) => { // chunk. The mapping allows us to efficiently lookup repository metadata. const createReposMapForChunk = async (chunk: ZoektGrpcSearchResponse, reposMapCache: Map, prisma: PrismaClient): Promise> => { const reposMap = new Map(); - await Promise.all(chunk.files.map(async (file) => { - const id = getRepoIdForFile(file); - + const repoIds = [...new Set(chunk.files.map(getRepoIdForFile))]; + await Promise.all(repoIds.map(async (id) => { const repo = await (async () => { // If it's in the cache, return the cached value. if (reposMapCache.has(id)) { From cde93da09923005394e5e07f434653cd06550d89 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Thu, 24 Sep 2026 00:12:37 -0400 Subject: [PATCH 2/2] chore: add changelog entry for #1684 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f4d08c7..0b4e773a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Made the default home page configurable with `DEFAULT_HOME_VIEW_PAGE`, defaulting to Code Search and supporting Ask. [#1677](https://github.com/sourcebot-dev/sourcebot/pull/1677) - Require authentication for the streaming and blocking Ask APIs in Public SaaS deployments. [#1679](https://github.com/sourcebot-dev/sourcebot/pull/1679) +- Fixed duplicate repository metadata lookups within search result chunks. [#1684](https://github.com/sourcebot-dev/sourcebot/pull/1684) ## [5.1.14] - 2026-09-17