From f51cc903a5c1f467948873f4f1873250d2b03258 Mon Sep 17 00:00:00 2001 From: julialeex Date: Mon, 24 Feb 2025 03:24:47 +0900 Subject: [PATCH] Fix(chat): prompt caching feature flag enrollement (#7177) (#7188) This commit fixes a bug in the prompt caching feature flag implementation where we were incorrectly using the enrollment event result instead of the actual feature flag value. This caused prompt caching to be enabled for all users regardless of their feature flag state. Key changes: - Added proper feature flag value tracking in PromptBuilder using a PromptCachingSetting interface - Separated the enrollment event logging from the feature flag evaluation - Ensures the feature flag value is correctly respected while still tracking enrollment events The fix maintains proper A/B testing capabilities while correctly respecting the intended feature flag distribution for prompt caching functionality. prompt caching should not be used for users who are not enrolled (non-dot com users) (cherry picked from commit 9cc241524c0bdfc09c2b6e62b118d9858ca8d57a) ## Test plan Co-authored-by: Beatrix <68532117+abeatrix@users.noreply.github.com> --- .../cody-chat_103640681/recording.har.yaml | 176 +++++++++++---- .../recording.har.yaml | 155 +++++++------- .../recording.har.yaml | 200 +++++++++--------- .../custom-commands.test.ts.snap | 9 +- .../__snapshots__/document-code.test.ts.snap | 6 +- agent/src/custom-commands.test.ts | 2 +- vscode/src/chat/chat-view/prompt.test.ts | 18 +- vscode/src/prompt-builder/index.test.ts | 38 ++++ vscode/src/prompt-builder/index.ts | 27 ++- 9 files changed, 381 insertions(+), 250 deletions(-) diff --git a/agent/recordings/cody-chat_103640681/recording.har.yaml b/agent/recordings/cody-chat_103640681/recording.har.yaml index aac87f52377e..bf28cc63bd6a 100644 --- a/agent/recordings/cody-chat_103640681/recording.har.yaml +++ b/agent/recordings/cody-chat_103640681/recording.har.yaml @@ -256,7 +256,7 @@ log: text: Ok. - speaker: human text: >- - Answer positively without apologizing. + Answer positively without apologizing. You have access to the provided codebase context. @@ -332,11 +332,11 @@ log: send: 0 ssl: -1 wait: 1087 - - _id: 41376641b9273b25289527160cfac275 + - _id: 4e18273d8ae5cff5662ebe8b0c813b4e _order: 0 cache: {} request: - bodySize: 933 + bodySize: 463 cookies: [] headers: - name: accept-encoding @@ -354,64 +354,148 @@ log: value: cody-cli 6.0.0-SNAPSHOT - name: host value: sourcegraph.sourcegraph.com - headersSize: 511 + headersSize: 497 httpVersion: HTTP/1.1 method: POST postData: mimeType: application/json params: [] textJSON: - maxTokensToSample: 4000 + maxTokensToSample: 100 messages: - - speaker: system - text: >- - You are Cody, an AI coding assistant from Sourcegraph.If your - answer contains fenced code blocks in Markdown, include the - relevant full file path in the code block tag using this - structure: ```$LANGUAGE:$FILEPATH``` - - For executable terminal commands: enclose each command in individual "bash" language code block without comments and new lines inside. - - cacheEnabled: true - speaker: human - text: |- - Codebase context from file animal.ts: - ```typescript:animal.ts - interface StrangeAnimal { - makesSound(): 'coo' | 'moo' - } - ``` - - speaker: assistant - text: Ok. - speaker: human text: >- - Answer positively without apologizing. + You are Cody, an AI coding assistant from Sourcegraph. Your + task is to generate a concise title (in about 10 words without + quotation) for respond with "hello" and nothing + else. + RULE: Your response should only contain the concise title and nothing else. + - speaker: assistant + model: google::v1::gemini-1.5-flash + temperature: 0.2 + topK: -1 + topP: -1 + queryString: + - name: client-name + value: jetbrains + - name: client-version + value: 6.0.0-SNAPSHOT + url: https://sourcegraph.sourcegraph.com/.api/completions/stream?client-name=jetbrains&client-version=6.0.0-SNAPSHOT + response: + bodySize: 170 + content: + mimeType: text/event-stream + size: 170 + text: >+ + event: completion + data: {"completion":"Simple \"Hello\" Response Function\n","stopReason":"STOP"} - You have access to the provided codebase context. + event: done - Question: implement a cow. Only print the code without any explanation. - model: anthropic::2024-10-22::claude-3-5-sonnet-latest + data: {} + + cookies: [] + headers: + - name: date + value: Fri, 21 Feb 2025 22:46:10 GMT + - name: content-type + value: text/event-stream + - name: transfer-encoding + value: chunked + - name: connection + value: keep-alive + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache + - name: vary + value: Accept-Encoding, Authorization, Cookie, Authorization, X-Requested-With, + Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1390 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2025-02-21T22:46:09.784Z + time: 1129 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 1129 + - _id: 1a932680a38d2702c78d0d744e3cc0a6 + _order: 0 + cache: {} + request: + bodySize: 485 + cookies: [] + headers: + - name: accept-encoding + value: gzip;q=0 + - name: authorization + value: token + REDACTED_1858aad0e1ff07ae26d4042086acb9da455866ad617afd2cb9ab9419e1be1104 + - name: connection + value: keep-alive + - name: content-type + value: application/json + - name: user-agent + value: cody-cli/6.0.0-SNAPSHOT (Node.js v20.4.0) + - name: x-requested-with + value: cody-cli 6.0.0-SNAPSHOT + - name: host + value: sourcegraph.sourcegraph.com + headersSize: 497 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json + params: [] + textJSON: + maxTokensToSample: 100 + messages: + - speaker: human + text: >- + You are Cody, an AI coding assistant from Sourcegraph. Your + task is to generate a concise title (in about 10 words without + quotation) for implement a cow. Only print the + code without any explanation.. + RULE: Your response should only contain the concise title and nothing else. + - speaker: assistant + model: google::v1::gemini-1.5-flash temperature: 0.2 topK: -1 topP: -1 queryString: - - name: api-version - value: "2" - name: client-name value: jetbrains - name: client-version value: 6.0.0-SNAPSHOT - url: https://sourcegraph.sourcegraph.com/.api/completions/stream?api-version=2&client-name=jetbrains&client-version=6.0.0-SNAPSHOT + url: https://sourcegraph.sourcegraph.com/.api/completions/stream?client-name=jetbrains&client-version=6.0.0-SNAPSHOT response: - bodySize: 584 + bodySize: 162 content: mimeType: text/event-stream - size: 584 + size: 162 text: >+ event: completion - data: {"deltaText":"```typescript:animal.ts\nclass Cow implements StrangeAnimal {\n makesSound(): 'coo' | 'moo' {\n return 'moo'\n }\n}\n```","stopReason":"end_turn"} + data: {"completion":"Cow Implementation: Code Only\n","stopReason":"STOP"} event: done @@ -421,7 +505,7 @@ log: cookies: [] headers: - name: date - value: Mon, 10 Feb 2025 19:57:13 GMT + value: Fri, 21 Feb 2025 22:46:10 GMT - name: content-type value: text/event-stream - name: transfer-encoding @@ -450,8 +534,8 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-10T19:57:12.335Z - time: 1562 + startedDateTime: 2025-02-21T22:46:09.883Z + time: 958 timings: blocked: -1 connect: -1 @@ -459,7 +543,7 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1562 + wait: 958 - _id: 5d0e29776bf0857beffb22b778ce488b _order: 0 cache: {} @@ -499,7 +583,7 @@ log: params: [] textJSON: query: > - + query CodeSearchEnabled { codeSearchEnabled: enterpriseLicenseHasFeature(feature:"code-search") } @@ -592,7 +676,7 @@ log: params: [] textJSON: query: |- - + query ContextFilters { site { codyContextFilters(version: V1) { @@ -700,7 +784,7 @@ log: params: [] textJSON: query: |- - + query CurrentSiteCodyLlmConfiguration { site { codyLLMConfiguration { @@ -820,7 +904,7 @@ log: params: [] textJSON: query: |- - + query CurrentSiteCodyLlmConfiguration { site { codyLLMConfiguration { @@ -928,7 +1012,7 @@ log: params: [] textJSON: query: |- - + query CurrentSiteCodyLlmProvider { site { codyLLMConfiguration { @@ -1036,7 +1120,7 @@ log: params: [] textJSON: query: |- - + query CurrentUser { currentUser { id @@ -1163,7 +1247,7 @@ log: params: [] textJSON: query: |- - + query SiteProductVersion { site { productVersion @@ -1273,7 +1357,7 @@ log: params: [] textJSON: query: | - + query ViewerSettings { viewerSettings { final diff --git a/agent/recordings/customCommandsClient_509552979/recording.har.yaml b/agent/recordings/customCommandsClient_509552979/recording.har.yaml index d30dc79724fa..3d1218f3e763 100644 --- a/agent/recordings/customCommandsClient_509552979/recording.har.yaml +++ b/agent/recordings/customCommandsClient_509552979/recording.har.yaml @@ -350,11 +350,11 @@ log: send: 0 ssl: -1 wait: 808 - - _id: 7f72c116f78d7588d22fe948c059381b + - _id: 4b3a65cb083f2c0c16a80a7105baeeb3 _order: 0 cache: {} request: - bodySize: 1768 + bodySize: 2126 cookies: [] headers: - name: accept-encoding @@ -367,7 +367,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-aeba78fadea4c98c4a53e0743172074b-b59daec458ea02cb-01 + value: 00-f8aafe08143b0bb2b215a4bddb5936fa-b3d4dc52cfb7d927-01 - name: user-agent value: customcommandsclient/v1 (Node.js v20.4.0) - name: x-requested-with @@ -391,13 +391,50 @@ log: structure: ```$LANGUAGE:$FILEPATH``` For executable terminal commands: enclose each command in individual "bash" language code block without comments and new lines inside. - - cacheEnabled: true - speaker: human + - speaker: human text: > - My selected code from codebase file src/animal.ts:1-6: - - ``` - + Codebase context from file path src/sum.ts: export function + sum(a: number, b: number): number { + /* CURSOR */ + } + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/example4.ts: export + function example(): string { + return 'example' + } + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/example3.ts: export + function example(): string { + return 'example' + } + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/example2.ts: export + function example(): string { + return 'example' + } + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/example1.ts: export + function example(): string { + return 'example' + } + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/animal.ts: /* + SELECTION_START */ export interface Animal { name: string @@ -405,48 +442,20 @@ log: isMammal: boolean } + /* SELECTION_END */ + - speaker: assistant + text: Ok. + - speaker: human + text: |- + My selected code from codebase file src/animal.ts:1-6: ``` - - Codebase context from file path src/animal.ts: /* SELECTION_START */ - export interface Animal { name: string makeAnimalSound(): string isMammal: boolean } - - /* SELECTION_END */ - - - - Codebase context from file path src/example1.ts: export function example(): string { - return 'example' - } - - - - Codebase context from file path src/example2.ts: export function example(): string { - return 'example' - } - - - - Codebase context from file path src/example3.ts: export function example(): string { - return 'example' - } - - - - Codebase context from file path src/example4.ts: export function example(): string { - return 'example' - } - - - - Codebase context from file path src/sum.ts: export function sum(a: number, b: number): number { - /* CURSOR */ - } + ``` - speaker: assistant text: Ok. - speaker: human @@ -477,7 +486,7 @@ log: size: 116 text: |+ event: completion - data: {"deltaText":"6","stopReason":"end_turn"} + data: {"deltaText":"7","stopReason":"end_turn"} event: done data: {} @@ -485,7 +494,7 @@ log: cookies: [] headers: - name: date - value: Mon, 10 Feb 2025 19:57:13 GMT + value: Fri, 21 Feb 2025 22:46:11 GMT - name: content-type value: text/event-stream - name: transfer-encoding @@ -514,8 +523,8 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-10T19:57:11.529Z - time: 1766 + startedDateTime: 2025-02-21T22:46:09.511Z + time: 2457 timings: blocked: -1 connect: -1 @@ -523,12 +532,12 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1766 - - _id: c9c5b45f95029b259a84cd726de59a01 + wait: 2457 + - _id: 6bf2501884491cd1c6f6f52c22eaba8f _order: 0 cache: {} request: - bodySize: 2263 + bodySize: 2243 cookies: [] headers: - name: accept-encoding @@ -541,7 +550,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-89c8a378d827e06424ac34ea658fe332-ae5725fefad9db64-01 + value: 00-e921341a329062630398c1b0bc5c251a-233a1f539fd750a0-01 - name: user-agent value: customcommandsclient/v1 (Node.js v20.4.0) - name: x-requested-with @@ -579,8 +588,7 @@ log: - Only enclose your response in XML tags. Do use any other XML tags unless they are part of the generated code. - Do not provide any additional commentary about the changes you made. Only respond with the generated code. - - cacheEnabled: true - speaker: human + - speaker: human text: |- My selected code from codebase file src/sum.ts:1-3: ``` @@ -644,15 +652,13 @@ log: cookies: [] headers: - name: date - value: Tue, 11 Feb 2025 18:54:44 GMT + value: Fri, 21 Feb 2025 22:46:13 GMT - name: content-type value: text/event-stream - name: transfer-encoding value: chunked - name: connection value: keep-alive - - name: retry-after - value: "67" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -670,13 +676,13 @@ log: value: 1; mode=block - name: strict-transport-security value: max-age=31536000; includeSubDomains; preload - headersSize: 1405 + headersSize: 1299 httpVersion: HTTP/1.1 redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-11T18:54:42.446Z - time: 1894 + startedDateTime: 2025-02-21T22:46:12.012Z + time: 1310 timings: blocked: -1 connect: -1 @@ -684,12 +690,12 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1894 - - _id: 82aefa70a5ee53eee9a9b81eefc59c60 + wait: 1310 + - _id: 4a1005f2f803fcf7e22b9b3a2d322c6e _order: 0 cache: {} request: - bodySize: 2381 + bodySize: 2361 cookies: [] headers: - name: accept-encoding @@ -702,7 +708,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-00ff2202f4b178a0c3ada66f2d21ba34-48a16ab077ec4be1-01 + value: 00-fb5564ef4e5d5089f74fd084794e3380-82fdc0dc975831d5-01 - name: user-agent value: customcommandsclient/v1 (Node.js v20.4.0) - name: x-requested-with @@ -740,8 +746,7 @@ log: - Only enclose your response in XML tags. Do use any other XML tags unless they are part of the generated code. - Do not provide any additional commentary about the changes you made. Only respond with the generated code. - - cacheEnabled: true - speaker: human + - speaker: human text: > Codebase context from file path src/animal.ts: /* SELECTION_START */ @@ -801,14 +806,14 @@ log: value: v1 url: https://sourcegraph.com/.api/completions/stream?api-version=2&client-name=customcommandsclient&client-version=v1 response: - bodySize: 451 + bodySize: 587 content: mimeType: text/event-stream - size: 451 + size: 587 text: >+ event: completion - data: {"deltaText":"export interface Animal {\n name: string\n makeAnimalSound(): string\n isMammal: boolean\n logName(): void\n}","stopReason":"stop_sequence"} + data: {"deltaText":"/* SELECTION_START */\nexport interface Animal {\n name: string\n makeAnimalSound(): string\n isMammal: boolean\n logAnimalName(): void\n}\n/* SELECTION_END */\n","stopReason":"stop_sequence"} event: done @@ -818,15 +823,13 @@ log: cookies: [] headers: - name: date - value: Tue, 11 Feb 2025 18:54:46 GMT + value: Fri, 21 Feb 2025 22:46:15 GMT - name: content-type value: text/event-stream - name: transfer-encoding value: chunked - name: connection value: keep-alive - - name: retry-after - value: "65" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -844,13 +847,13 @@ log: value: 1; mode=block - name: strict-transport-security value: max-age=31536000; includeSubDomains; preload - headersSize: 1405 + headersSize: 1299 httpVersion: HTTP/1.1 redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-11T18:54:44.359Z - time: 1762 + startedDateTime: 2025-02-21T22:46:13.353Z + time: 2186 timings: blocked: -1 connect: -1 @@ -858,7 +861,7 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1762 + wait: 2186 - _id: f1b1cde4cd57488b7f0137954f0302f5 _order: 0 cache: {} diff --git a/agent/recordings/document-code_965949506/recording.har.yaml b/agent/recordings/document-code_965949506/recording.har.yaml index 5f8b8c7c8244..f7d64cee9bd4 100644 --- a/agent/recordings/document-code_965949506/recording.har.yaml +++ b/agent/recordings/document-code_965949506/recording.har.yaml @@ -449,11 +449,11 @@ log: send: 0 ssl: -1 wait: 1297 - - _id: c2cd55110c5891dd44665a65b9e6eae4 + - _id: d4b7126134fbb6b0bd8e84cd9391d7c5 _order: 0 cache: {} request: - bodySize: 3004 + bodySize: 2572 cookies: [] headers: - name: accept-encoding @@ -466,7 +466,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-bc1c2779927b127f8ce19d458438e735-141708e418ae2c04-01 + value: 00-556b4851df49b95f96f6d4397a4ffc67-5b4f84e8b0fcb556-01 - name: user-agent value: document-code/v1 (Node.js v20.4.0) - name: x-requested-with @@ -504,38 +504,19 @@ log: - Only enclose your response in XML tags. Do use any other XML tags unless they are part of the generated code. - Do not provide any additional commentary about the changes you made. Only respond with the generated code. - - cacheEnabled: true - speaker: human - text: >+ - Codebase context from file path src/TestLogger.ts: Codebase - context from file src/TestLogger.ts: - - recordLog() - }, - } - - - - Codebase context from file path src/TestLogger.ts: Codebase context from file src/TestLogger.ts: - - const foo = 42 - - export const TestLogger = { - startLogging: () => { - // Do some stuff - - - speaker: assistant - text: Ok. - speaker: human text: >- - This is part of the file: src/TestLogger.ts + This is part of the file: src/Hello.kt The user has the following code in their selection: - function recordLog() { - console.log(/* CURSOR */ 'Recording the log') - } + class He/* CURSOR */llo { + fun greeting(): String { + return "Hello, world!" + } + } + @@ -553,7 +534,7 @@ log: model: anthropic/claude-3-haiku-20240307 stopSequences: - - - " function recordLog() {" + - class He/* CURSOR */llo { temperature: 0 topK: -1 topP: -1 @@ -566,14 +547,14 @@ log: value: v1 url: https://sourcegraph.com/.api/completions/stream?api-version=2&client-name=document-code&client-version=v1 response: - bodySize: 318 + bodySize: 306 content: mimeType: text/event-stream - size: 318 + size: 306 text: >+ event: completion - data: {"deltaText":"\n/**\n * Logs a message indicating that a log is being recorded.\n */\n","stopReason":"stop_sequence"} + data: {"deltaText":"/**\n * Provides a simple \"Hello, world!\" greeting.\n */\n","stopReason":"stop_sequence"} event: done @@ -583,7 +564,7 @@ log: cookies: [] headers: - name: date - value: Tue, 11 Feb 2025 18:54:48 GMT + value: Tue, 11 Feb 2025 18:54:50 GMT - name: content-type value: text/event-stream - name: transfer-encoding @@ -591,7 +572,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "62" + value: "60" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -614,8 +595,8 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-11T18:54:47.434Z - time: 862 + startedDateTime: 2025-02-11T18:54:49.626Z + time: 1372 timings: blocked: -1 connect: -1 @@ -623,12 +604,12 @@ log: receive: 0 send: 0 ssl: -1 - wait: 862 - - _id: 7c6bfb662f305f30b728800a5f464d08 + wait: 1372 + - _id: d94ffaa46d88c0fbdeba278c66405b04 _order: 0 cache: {} request: - bodySize: 3257 + bodySize: 3047 cookies: [] headers: - name: accept-encoding @@ -641,7 +622,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-e8a4860176977ed241ec85bf791e689c-b3ec88bee0ad8b01-01 + value: 00-54c36bcee75e9407a85dc2378150ef9c-c364897e60ee971b-01 - name: user-agent value: document-code/v1 (Node.js v20.4.0) - name: x-requested-with @@ -679,47 +660,39 @@ log: - Only enclose your response in XML tags. Do use any other XML tags unless they are part of the generated code. - Do not provide any additional commentary about the changes you made. Only respond with the generated code. - - cacheEnabled: true - speaker: human - text: > - Codebase context from file path src/example.test.ts: Codebase - context from file src/example.test.ts: - }) - }) - - - - Codebase context from file path src/example.test.ts: Codebase context from file src/example.test.ts: - - import { expect } from 'vitest' - - import { it } from 'vitest' - - import { describe } from 'vitest' + - speaker: human + text: >+ + Codebase context from file path src/TestLogger.ts: Codebase + context from file src/TestLogger.ts: + const foo = 42 - describe('test block', () => { - it('does 1', () => { - expect(true).toBe(true) - }) + export const TestLogger = { + startLogging: () => { + // Do some stuff - it('does 2', () => { - expect(true).toBe(true) - }) + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/TestLogger.ts: Codebase + context from file src/TestLogger.ts: - it('does something else', () => { - // This line will error due to incorrect usage of `performance.now` + recordLog() + }, + } - speaker: assistant text: Ok. - speaker: human text: >- - This is part of the file: src/example.test.ts + This is part of the file: src/TestLogger.ts The user has the following code in their selection: - const startTime = performance.now(/* CURSOR */) - + function recordLog() { + console.log(/* CURSOR */ 'Recording the log') + } @@ -737,7 +710,7 @@ log: model: anthropic/claude-3-haiku-20240307 stopSequences: - - - " const startTime = performance.now(/* CURSOR */)" + - " function recordLog() {" temperature: 0 topK: -1 topP: -1 @@ -750,14 +723,14 @@ log: value: v1 url: https://sourcegraph.com/.api/completions/stream?api-version=2&client-name=document-code&client-version=v1 response: - bodySize: 446 + bodySize: 216 content: mimeType: text/event-stream - size: 446 + size: 216 text: >+ event: completion - data: {"deltaText":"\n/**\n * Retrieves the current time in milliseconds since the page was loaded.\n * This is useful for measuring the duration of an operation or event.\n */\n","stopReason":"stop_sequence"} + data: {"deltaText":"\n/**\n * Records a log message to the console.\n */\n","stopReason":"stop_sequence"} event: done @@ -767,15 +740,13 @@ log: cookies: [] headers: - name: date - value: Tue, 11 Feb 2025 18:54:49 GMT + value: Fri, 21 Feb 2025 22:46:16 GMT - name: content-type value: text/event-stream - name: transfer-encoding value: chunked - name: connection value: keep-alive - - name: retry-after - value: "61" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -793,13 +764,13 @@ log: value: 1; mode=block - name: strict-transport-security value: max-age=31536000; includeSubDomains; preload - headersSize: 1405 + headersSize: 1299 httpVersion: HTTP/1.1 redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-11T18:54:48.312Z - time: 1252 + startedDateTime: 2025-02-21T22:46:15.078Z + time: 1066 timings: blocked: -1 connect: -1 @@ -807,12 +778,12 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1252 - - _id: d4b7126134fbb6b0bd8e84cd9391d7c5 + wait: 1066 + - _id: 11242d3d661836d08d91de36ee8dd1ce _order: 0 cache: {} request: - bodySize: 2572 + bodySize: 3300 cookies: [] headers: - name: accept-encoding @@ -825,7 +796,7 @@ log: - name: content-type value: application/json - name: traceparent - value: 00-556b4851df49b95f96f6d4397a4ffc67-5b4f84e8b0fcb556-01 + value: 00-21d9b2e4143b07399e332d57c2dfae91-9a353f9548f25562-01 - name: user-agent value: document-code/v1 (Node.js v20.4.0) - name: x-requested-with @@ -863,18 +834,47 @@ log: - Only enclose your response in XML tags. Do use any other XML tags unless they are part of the generated code. - Do not provide any additional commentary about the changes you made. Only respond with the generated code. + - speaker: human + text: > + Codebase context from file path src/example.test.ts: Codebase + context from file src/example.test.ts: + + import { expect } from 'vitest' + + import { it } from 'vitest' + + import { describe } from 'vitest' + + + describe('test block', () => { + it('does 1', () => { + expect(true).toBe(true) + }) + + it('does 2', () => { + expect(true).toBe(true) + }) + + it('does something else', () => { + // This line will error due to incorrect usage of `performance.now` + - speaker: assistant + text: Ok. + - speaker: human + text: > + Codebase context from file path src/example.test.ts: Codebase + context from file src/example.test.ts: + }) + }) + - speaker: assistant + text: Ok. - speaker: human text: >- - This is part of the file: src/Hello.kt + This is part of the file: src/example.test.ts The user has the following code in their selection: - class He/* CURSOR */llo { - fun greeting(): String { - return "Hello, world!" - } - } + const startTime = performance.now(/* CURSOR */) @@ -893,7 +893,7 @@ log: model: anthropic/claude-3-haiku-20240307 stopSequences: - - - class He/* CURSOR */llo { + - " const startTime = performance.now(/* CURSOR */)" temperature: 0 topK: -1 topP: -1 @@ -906,14 +906,14 @@ log: value: v1 url: https://sourcegraph.com/.api/completions/stream?api-version=2&client-name=document-code&client-version=v1 response: - bodySize: 306 + bodySize: 425 content: mimeType: text/event-stream - size: 306 + size: 425 text: >+ event: completion - data: {"deltaText":"/**\n * Provides a simple \"Hello, world!\" greeting.\n */\n","stopReason":"stop_sequence"} + data: {"deltaText":"\n/**\n * Captures the current timestamp using `performance.now()`.\n * This is useful for measuring the duration of an operation.\n */\n","stopReason":"stop_sequence"} event: done @@ -923,15 +923,13 @@ log: cookies: [] headers: - name: date - value: Tue, 11 Feb 2025 18:54:50 GMT + value: Fri, 21 Feb 2025 22:46:17 GMT - name: content-type value: text/event-stream - name: transfer-encoding value: chunked - name: connection value: keep-alive - - name: retry-after - value: "60" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -949,13 +947,13 @@ log: value: 1; mode=block - name: strict-transport-security value: max-age=31536000; includeSubDomains; preload - headersSize: 1405 + headersSize: 1299 httpVersion: HTTP/1.1 redirectURL: "" status: 200 statusText: OK - startedDateTime: 2025-02-11T18:54:49.626Z - time: 1372 + startedDateTime: 2025-02-21T22:46:16.168Z + time: 1530 timings: blocked: -1 connect: -1 @@ -963,7 +961,7 @@ log: receive: 0 send: 0 ssl: -1 - wait: 1372 + wait: 1530 - _id: f1b1cde4cd57488b7f0137954f0302f5 _order: 0 cache: {} diff --git a/agent/src/__snapshots__/custom-commands.test.ts.snap b/agent/src/__snapshots__/custom-commands.test.ts.snap index 4f0050e9e0dc..c2ee2a39df87 100644 --- a/agent/src/__snapshots__/custom-commands.test.ts.snap +++ b/agent/src/__snapshots__/custom-commands.test.ts.snap @@ -8,12 +8,15 @@ exports[`Custom Commands > commands/custom, chat command, open tabs context 1`] `; exports[`Custom Commands > commands/custom, edit command, edit mode 1`] = ` -"export interface Animal { +"/* SELECTION_START */ +export interface Animal { name: string makeAnimalSound(): string isMammal: boolean - logName(): void -}" + logAnimalName(): void +} +/* SELECTION_END */ +" `; exports[`Custom Commands > commands/custom, edit command, insert mode 1`] = ` diff --git a/agent/src/__snapshots__/document-code.test.ts.snap b/agent/src/__snapshots__/document-code.test.ts.snap index 694fbbf04af2..d59735ecd701 100644 --- a/agent/src/__snapshots__/document-code.test.ts.snap +++ b/agent/src/__snapshots__/document-code.test.ts.snap @@ -7,7 +7,7 @@ export const TestLogger = { // Do some stuff /** - * Logs a message indicating that a log is being recorded. + * Records a log message to the console. */ function recordLog() { console.log(/* CURSOR */ 'Recording the log') @@ -87,8 +87,8 @@ describe('test block', () => { it('does something else', () => { // This line will error due to incorrect usage of \`performance.now\` /** - * Retrieves the current time in milliseconds since the page was loaded. - * This is useful for measuring the duration of an operation or event. + * Captures the current timestamp using \`performance.now()\`. + * This is useful for measuring the duration of an operation. */ const startTime = performance.now(/* CURSOR */) }) diff --git a/agent/src/custom-commands.test.ts b/agent/src/custom-commands.test.ts index 2aeac543116b..032b6be4cfc9 100644 --- a/agent/src/custom-commands.test.ts +++ b/agent/src/custom-commands.test.ts @@ -82,7 +82,7 @@ describe('Custom Commands', () => { expect(result.type).toBe('chat') const lastMessage = await client.firstNonEmptyTranscript(result.chatResult as string) const reply = trimEndOfLine(lastMessage.messages.at(-1)?.text ?? '') - expect(reply).toMatchInlineSnapshot(`"6"`, explainPollyError) + expect(reply).toMatchInlineSnapshot(`"7"`, explainPollyError) }, 30_000) it('commands/custom, edit command, insert mode', async () => { diff --git a/vscode/src/chat/chat-view/prompt.test.ts b/vscode/src/chat/chat-view/prompt.test.ts index 18f11d8ef2bb..bb5aefd79a83 100644 --- a/vscode/src/chat/chat-view/prompt.test.ts +++ b/vscode/src/chat/chat-view/prompt.test.ts @@ -3,6 +3,7 @@ import { CLIENT_CAPABILITIES_FIXTURE, type ContextItem, ContextItemSource, + FeatureFlag, type Message, ModelUsage, type ModelsData, @@ -43,18 +44,11 @@ describe('DefaultPrompter', () => { }) vi.spyOn(localStorage, 'getEnrollmentHistory').mockReturnValue(false) vi.spyOn(contextFiltersProvider, 'isUriIgnored').mockResolvedValue(false) - vi.spyOn(graphqlClient, 'fetchSourcegraphAPI').mockImplementation(async () => ({ - data: { - evaluateFeatureFlag: true, - featureFlags: { - evaluatedFeatureFlags: [ - { name: 'cody-intent-detection-api', value: true }, - { name: 'cody-unified-prompts', value: true }, - { name: 'cody-autocomplete-tracing', value: true }, - ], - }, - }, - })) + vi.spyOn(graphqlClient, 'getEvaluatedFeatureFlags').mockResolvedValue({ + [FeatureFlag.CodyUnifiedPrompts]: true, + [FeatureFlag.CodyAutocompleteTracing]: true, + [FeatureFlag.CodyPromptCachingOnMessages]: false, + }) mockAuthStatus(AUTH_STATUS_FIXTURE_AUTHED) mockClientCapabilities(CLIENT_CAPABILITIES_FIXTURE) }) diff --git a/vscode/src/prompt-builder/index.test.ts b/vscode/src/prompt-builder/index.test.ts index bdf26b4d65d3..23efb2cae504 100644 --- a/vscode/src/prompt-builder/index.test.ts +++ b/vscode/src/prompt-builder/index.test.ts @@ -4,8 +4,10 @@ import { ContextItemSource, contextFiltersProvider, displayPath, + featureFlagProvider, ps, } from '@sourcegraph/cody-shared' +import { Observable } from 'observable-fns' import { beforeEach, describe, expect, it, vi } from 'vitest' import { URI } from 'vscode-uri' import { mockLocalStorage } from '../services/LocalStorageProvider' @@ -386,3 +388,39 @@ describe('PromptBuilder', () => { }) }) }) + +describe('PromptBuilder', () => { + beforeEach(() => { + vi.spyOn(featureFlagProvider, 'evaluatedFeatureFlag').mockReturnValue(Observable.of(false)) + }) + describe('isCacheEnabled', () => { + it('handles disabled feature flag correctly', async () => { + const promptBuilder = await PromptBuilder.create({ + input: 8192, + output: 4096, + }) + + // First access should trigger enrollment but still return feature flag value + expect(promptBuilder.isCacheEnabled).toBe(false) + + // Second access should use cached value + expect(promptBuilder.isCacheEnabled).toBe(false) + }) + + it('respects feature flag value and tracks enrollment', async () => { + // Mock feature flag provider + vi.spyOn(featureFlagProvider, 'evaluatedFeatureFlag').mockReturnValue(Observable.of(true)) + const promptBuilder = await PromptBuilder.create({ + input: 8192, + output: 4096, + }) + + // First access should trigger enrollment but still return feature flag value + expect(promptBuilder.isCacheEnabled).toBe(true) + + // Second access should use cached value + expect(promptBuilder.isCacheEnabled).toBe(true) + vi.spyOn(featureFlagProvider, 'evaluatedFeatureFlag').mockReturnValue(Observable.of(false)) + }) + }) +}) diff --git a/vscode/src/prompt-builder/index.ts b/vscode/src/prompt-builder/index.ts index aec215d668c7..1b2d769e9c1d 100644 --- a/vscode/src/prompt-builder/index.ts +++ b/vscode/src/prompt-builder/index.ts @@ -23,6 +23,11 @@ interface PromptBuilderContextResult { added: ContextItem[] } +interface PromptCachingSetting { + featureFlag?: boolean + isEnrolled?: boolean +} + const ASSISTANT_MESSAGE = { speaker: 'assistant', text: ps`Ok.` } as Message /** * PromptBuilder constructs a full prompt given a charLimit constraint. @@ -48,17 +53,23 @@ export class PromptBuilder { private constructor(private readonly tokenCounter: TokenCounter) {} - private _isCacheEnabled: boolean | undefined - private get isCacheEnabled(): boolean { - if (this._isCacheEnabled === undefined) { - this._isCacheEnabled = logFirstEnrollmentEvent( + private readonly hasCacheFeatureFlag = storeLastValue( + featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyPromptCachingOnMessages) + ) + private readonly _isCacheEnabled: PromptCachingSetting = { + featureFlag: false, + isEnrolled: false, + } + + public get isCacheEnabled(): boolean { + const isFlagEnabled = Boolean(this.hasCacheFeatureFlag?.value?.last ?? false) + if (!this._isCacheEnabled.isEnrolled) { + this._isCacheEnabled.isEnrolled = logFirstEnrollmentEvent( FeatureFlag.CodyPromptCachingOnMessages, - !!storeLastValue( - featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyPromptCachingOnMessages) - ).value + isFlagEnabled ) } - return this._isCacheEnabled + return isFlagEnabled } public build(): Message[] {