Merge pull request #37 from checkly/validation

Add validation to prompt functions

Author: jan-osch

src/prompts/alerts.ts

@@ -1,11 +1,16 @@
 import { getOpenaiSDKClient } from "../ai/openai";
 import { NotionPage } from "../notion/notion";
 import { PromptConfig, promptConfig } from "./common";
+import { getOpenaiSDKClient } from "../ai/openai";
+import { validObjectList, validString } from "./validation";
 
 export function affectedComponentsPrompt(
   guidelines: NotionPage[], // guidelines for ops channel
   alertMessage: string,
 ): [string, PromptConfig] {
+  validObjectList.parse(guidelines);
+  validString.parse(alertMessage);
+
   const model = getOpenaiSDKClient()("gpt-4o-mini");
   const systemPrompt = `You are an experienced on-call engineer who is responsible for determining which system components are affected by an alert`;
 
@@ -32,6 +37,10 @@ export function alertRecommendationPrompt(
   alertMessage: string,
   affectedComponents: { component: string; environment: string }[],
 ): [string, PromptConfig] {
+  validObjectList.parse(guidelines);
+  validString.parse(alertMessage);
+  validObjectList.parse(affectedComponents);
+
   const model = getOpenaiSDKClient()("gpt-4o-mini");
   const system =
     "You are an experienced on-call engineer who is responsible for recommending further actions for an alert";
@@ -62,6 +71,9 @@ export function alertHistoryPrompt(
   alertMessage: string,
   messageHistory: string,
 ): [string, PromptConfig] {
+  validString.parse(alertMessage);
+  validString.parse(messageHistory);
+
   const system =
     "You are an experienced on-call engineer who is responsible for analysing previous alert in the slack channel";
 
@@ -96,6 +108,9 @@ export function alertSeverityPrompt(
   alertMessage: string,
   affectedComponents: { component: string; environment: string }[],
 ): [string, PromptConfig] {
+  validString.parse(alertMessage);
+  validObjectList.parse(affectedComponents);
+
   const system =
     "You are an experienced on-call engineer who is responsible for determining the severity of alerts";
 
@@ -131,6 +146,17 @@ export function alertSummaryPrompt(
   historyInfo: { type: string; reasoning: string },
   guidelines: NotionPage[],
 ): [string, PromptConfig] {
+  validObjectList.parse(affectedComponents);
+  [
+    alertMessage,
+    severityInfo.severity,
+    severityInfo.reasoning,
+    stateInfo.state,
+    stateInfo.reasoning,
+    historyInfo.type,
+    historyInfo.reasoning,
+  ].forEach((s) => validString.parse(s));
+
   const system =
     "You are an experienced on-call engineer who is leading a team of engineers analysing alerts from a Slack channel";
 

src/prompts/checkly.ts

@@ -2,11 +2,11 @@ import { stringify } from "yaml";
 import { CheckContext, ContextKey } from "../aggregator/ContextAggregator";
 import { Check } from "../checkly/models";
 import { mapCheckToContextValue } from "../checkly/utils";
+import { validObjectList, validObject } from "./validation";
 import { promptConfig, PromptConfig } from "./common";
 import { slackFormatInstructions } from "./slack";
 
 /** Maximum length for context analysis text to prevent oversized prompts */
-
 const CONTEXT_ANALYSIS_MAX_LENGTH = 200000;
 
 /**
@@ -24,6 +24,9 @@ export function contextAnalysisEntryPrompt(
   entry: CheckContext,
   allEntries: CheckContext[],
 ): [string, PromptConfig] {
+  validObject.parse(entry);
+  validObjectList.parse(allEntries);
+
   return [
     `The following check has failed: ${formatChecklyMonitorData(allEntries)}
 
@@ -50,6 +53,8 @@ ${stringify(entry)}`,
 export function contextAnalysisSummaryPrompt(
   contextRows: CheckContext[],
 ): [string, PromptConfig] {
+  validObjectList.parse(contextRows);
+
   const checkContext = formatChecklyMonitorData(contextRows);
 
   return [
@@ -95,6 +100,8 @@ export function checklyToolPrompt(
   checks: Check[],
   query: string | undefined,
 ): [string, PromptConfig] {
+  validObjectList.parse(checks);
+
   return [
     `You are the Checkly Check Search Engine. You are given a query and a list of checks. Return the most relevant check that relates to the query.
 
@@ -117,6 +124,8 @@ Search Query: ${query ?? ""}`,
  * const formattedContext = formatContextAnalysis(contextRows);
  */
 function formatContextAnalysis(rows: CheckContext[]): string {
+  validObjectList.parse(rows);
+
   return stringify(
     rows
       .filter((c) => c.key !== ContextKey.ChecklyCheck)
@@ -136,6 +145,8 @@ function formatContextAnalysis(rows: CheckContext[]): string {
  * const monitorData = formatChecklyMonitorData(contextRows);
  */
 function formatChecklyMonitorData(rows: CheckContext[]): string {
+  validObjectList.parse(rows);
+
   const checkContext = rows.find((c) => c.key === ContextKey.ChecklyCheck);
   return stringify(
     {

src/prompts/github.ts

@@ -2,6 +2,7 @@ import { stringify } from "yaml";
 import { Check } from "../checkly/models";
 import { mapCheckToContextValue } from "../checkly/utils";
 import { PromptConfig, promptConfig } from "./common";
+import { validObjectList, validObject, validString } from "./validation";
 
 const MAX_DIFF_LENGTH = 1000000;
 
@@ -30,6 +31,10 @@ export function generateFindRelevantReleasesPrompt(
   checkResult: string,
   releases: GithubReleaseForPrompt[],
 ): [string, PromptConfig] {
+  validObject.parse(check);
+  validString.parse(checkResult);
+  validObjectList.parse(releases);
+
   return [
     `Based on the following releases, which ones are most relevant to the check state change?
 
@@ -61,6 +66,10 @@ export function generateFindRelevantDeploymentsPrompt(
   checkResult: string,
   deployments: GithubDeploymentForPrompt[],
 ): [string, PromptConfig] {
+  validObject.parse(check);
+  validString.parse(checkResult);
+  validObjectList.parse(deployments);
+
   return [
     `Based on the following deployments, which ones are most relevant to the check state change? Analyze the check script, result and releases to determine which releases are most relevant. Provide a list of deployment ids that are most relevant to the check.
 
@@ -89,6 +98,10 @@ export function generateReleaseHeadlinePrompt(
   currentRelease: string,
   diff: string,
 ): [string, PromptConfig] {
+  validString.parse(prevRelease);
+  validString.parse(currentRelease);
+  validString.parse(diff);
+
   return [
     `The following diff describes the changes between ${prevRelease} and ${currentRelease}.
 
@@ -112,6 +125,10 @@ export function generateReleaseSummaryPrompt(
   currentRelease: string,
   diff: string,
 ): [string, PromptConfig] {
+  validString.parse(prevRelease);
+  validString.parse(currentRelease);
+  validString.parse(diff);
+
   return [
     `The following diff describes the changes between ${prevRelease} and ${currentRelease}.
 
@@ -137,6 +154,10 @@ export function generateDeploymentSummaryPrompt(
   currentSha: string,
   diff: string,
 ): [string, PromptConfig] {
+  validString.parse(prevSha);
+  validString.parse(currentSha);
+  validString.parse(diff);
+
   return [
     `The following diff describes the changes between ${prevSha} and ${currentSha}.
 
@@ -159,6 +180,9 @@ export function generateFindRepoPrompt(
   userPrompt: string,
   allRepos: GithubRepoForPrompt[],
 ): [string, PromptConfig] {
+  validString.parse(userPrompt);
+  validObjectList.parse(allRepos);
+
   return [
     `Based on the following prompt: ${userPrompt} and the list of repositories
 

src/prompts/search.ts

@@ -1,5 +1,6 @@
 import { JsonValue } from "@prisma/client/runtime/library";
 import { PromptConfig, promptConfig } from "./common";
+import { validObjectList, validString } from "./validation";
 
 interface SearchContextRowForPrompt {
   key: string;
@@ -10,6 +11,9 @@ export function searchContextPrompt(
   query: string,
   contextRows: SearchContextRowForPrompt[],
 ): [string, PromptConfig] {
+  validString.parse(query);
+  validObjectList.parse(contextRows);
+
   const config = promptConfig({
     maxTokens: 1000,
     experimental_telemetry: {

src/prompts/slack.ts

@@ -1,6 +1,7 @@
 import { WebhookAlertDto } from "../checkly/alertDTO";
 import { convertSlackTimestamp } from "../slackbot/utils";
 import { PromptConfig, promptConfig } from "./common";
+import { validObjectList, validObject, validString } from "./validation";
 
 export const slackFormatInstructions = `Format all output in Slack mrkdwn format.
 Generate Slack messages using the following style: *bold*, <link|text>, _italics_, > quote, \`code\`, \`\`\`code block\`\`\`.
@@ -31,6 +32,9 @@ export function channelSummaryPrompt(
   alert: WebhookAlertDto,
   messageHistory: SlackMsgForPrompt[],
 ): [string, PromptConfig] {
+  validObject.parse(alert);
+  validObjectList.parse(messageHistory);
+
   return [
     `You are a Slack channel context collector. Your task is to analyze the given message history based on a specific prompt and provide a concise summary of the relevant context.
 

src/prompts/sre-assistant.ts

@@ -1,10 +1,15 @@
 import { slackFormatInstructions } from "./slack";
+import { validString } from "./validation";
 
 export function generateSREAssistantPrompt(
   username: string,
   date: string,
   alertSummary: string,
 ): string {
+  validString.parse(username);
+  validString.parse(date);
+  validString.parse(alertSummary);
+
   return `You are an AI-powered SRE Bot designed to assist in real-time incident management. Your primary goal is to reduce Mean Time To Resolution (MTTR) by automatically aggregating and analyzing contextual data, providing actionable insights, and guiding first responders effectively.
 
 CONSTITUTION:

src/prompts/timeframe.ts

@@ -1,9 +1,12 @@
 import moment from "moment";
 import { PromptConfig, promptConfig } from "./common";
+import { validString } from "./validation";
 
 export function parseTimeframePrompt(
   timeframe: string,
 ): [string, PromptConfig] {
+  validString.parse(timeframe);
+
   return [
     `Parse the following timeframe into a precise timestamp range with  and interpretation.