LLM Prompt 的 A/B 测试

from langfuse import get_client
import random
from langfuse.openai import openai
 
# Requires environment variables for initialization
from langfuse import get_client
langfuse = get_client()
 
# Fetch prompt versions
prompt_a = langfuse.get_prompt("my-prompt-name", label="prod-a")
prompt_b = langfuse.get_prompt("my-prompt-name", label="prod-b")
 
# Randomly select version
selected_prompt = random.choice([prompt_a, prompt_b])
 
# Use in LLM call
response = openai.chat.completions.create(
    model="gpt-3.5-turbo",
    messages=[{"role": "user", "content": selected_prompt.compile(variable="value")}],
    # Link prompt to generation for analytics
    langfuse_prompt=selected_prompt
)
result_text = response.choices[0].message.content

import { LangfuseClient } from "@langfuse/client";
import { observeOpenAI } from "@langfuse/openai";
 
import OpenAI from "openai";
 
// Requires environment variables for initialization
const langfuse = new LangfuseClient();
 
// Create and wrap OpenAI client
const openai = observeOpenAI(new OpenAI());
 
// Fetch prompt versions
const promptA = await langfuse.prompt.get("my-prompt-name", {
  label: "prod-a",
});
const promptB = await langfuse.prompt.get("my-prompt-name", {
  label: "prod-b",
});
 
// Randomly select version
const selectedPrompt = Math.random() < 0.5 ? promptA : promptB;
 
// Use in LLM call
const completion = await openai.chat.completions.create({
  model: "gpt-3.5-turbo",
  messages: [
    {
      role: "user",
      content: selectedPrompt.compile({ variable: "value" }),
    },
  ],
  // Link prompt to generation for analytics
  langfusePrompt: selectedPrompt,
});
const resultText = completion.choices[0].message.content;