from mlflow.genai.scorers import Correctness

# 使用不同的 Judge 模型
Correctness(model="anthropic:/claude-4-opus")
Correctness(model="openai:/gpt-4o")
Code language: PHP (php)

from mlflow.genai.scorers import (
    Correctness,          # 答案正確性（需要 ground truth）
    RelevanceToQuery,     # 回答是否切題
    Completeness,         # 是否完整回答所有問題
    Fluency,              # 語法流暢度
    Safety,               # 是否包含有害內容
    RetrievalGroundedness, # 回答是否有根據（RAG 應用）
    RetrievalRelevance,   # 檢索結果是否相關
    Guidelines,           # 自訂指南檢查
)
Code language: PHP (php)

import mlflow
from mlflow.genai.scorers import Correctness, RelevanceToQuery

# 1. 準備評估資料集
eval_dataset = [
    {
        "inputs": {"question": "MLflow 可以管理 prompt 嗎？"},
        "expectations": {"expected_response": "可以，MLflow 提供 Prompt Registry 功能"},
    },
    {
        "inputs": {"question": "什麼是 Tracing？"},
        "expectations": {"expected_response": "Tracing 是追蹤 LLM 應用執行流程的功能"},
    },
]

# 2. 定義你的應用（這裡用簡單函數示範）
def my_app(question: str) -> str:
    # 實際上這裡會呼叫 LLM
    return "MLflow 提供完整的 prompt 管理功能"

# 3. 執行評估
results = mlflow.genai.evaluate(
    data=eval_dataset,
    predict_fn=my_app,
    scorers=[Correctness(), RelevanceToQuery()],
)
Code language: PHP (php)

{
    "inputs": {"question": "如何使用 Docker？"},
    "expectations": {
        "expected_response": "...",
        "key_concepts": ["container", "image", "dockerfile"]
    },
    "tags": {"topic": "devops", "difficulty": "beginner"},
}
Code language: JSON / JSON with Comments (json)

from mlflow.genai.scorers import Guidelines

# 定義你的評估指南
professional_tone = Guidelines(
    name="professional_tone",
    guidelines="回答必須保持專業語調，不使用口語或表情符號"
)

no_code_in_response = Guidelines(
    name="no_code",
    guidelines="回答中不應包含程式碼範例"
)

# 使用多個指南
results = mlflow.genai.evaluate(
    data=eval_dataset,
    predict_fn=my_app,
    scorers=[professional_tone, no_code_in_response],
)
Code language: PHP (php)

from mlflow.genai.judges import make_judge
from typing import Literal

# 建立一個評估「連貫性」的 Judge
coherence_judge = make_judge(
    name="coherence",
    instructions=(
        "評估回答是否連貫，保持一致的語調，並有清楚的邏輯流程。\n"
        "問題: {{ inputs }}\n"
        "回答: {{ outputs }}\n"
    ),
    feedback_value_type=Literal["coherent", "somewhat_coherent", "incoherent"],
    model="openai:/gpt-4o-mini",
)
Code language: PHP (php)

# 布林值：pass/fail
make_judge(..., feedback_value_type=bool)

# 數值評分
make_judge(..., feedback_value_type=int)    # 整數
make_judge(..., feedback_value_type=float)  # 浮點數

# 分類結果（推薦）
make_judge(..., feedback_value_type=Literal["good", "acceptable", "poor"])
Code language: PHP (php)

from mlflow.genai.judges import make_judge
from typing import Literal

factual_accuracy_judge = make_judge(
    name="factual_accuracy",
    instructions="""
    評估回答的事實正確性。

    參考資料: {{ expectations }}

    回答: {{ outputs }}

    檢查回答中的事實陳述是否與參考資料一致。
    - "accurate": 所有事實陳述都正確
    - "partially_accurate": 部分正確，部分錯誤或遺漏
    - "inaccurate": 主要事實錯誤或存在幻覺
    """,
    feedback_value_type=Literal["accurate", "partially_accurate", "inaccurate"],
    model="openai:/gpt-4o-mini",
)

# 使用自訂 Judge
results = mlflow.genai.evaluate(
    data=eval_dataset,
    predict_fn=my_app,
    scorers=[factual_accuracy_judge],
)
Code language: PHP (php)

import mlflow
from datetime import datetime, timedelta

# 取得過去 24 小時的 traces
yesterday = datetime.now() - timedelta(days=1)
traces = mlflow.search_traces(
    filter_string=f"timestamp > {int(yesterday.timestamp() * 1000)}"
)

# traces 是 pandas DataFrame
print(f"找到 {len(traces)} 筆 traces")
Code language: PHP (php)

from mlflow.genai.scorers import RetrievalGroundedness, RelevanceToQuery, Guidelines

# 定義 Scorers
scorers = [
    RetrievalGroundedness(),
    RelevanceToQuery(),
    Guidelines(
        name="professional_tone",
        guidelines="回答必須保持專業語調"
    ),
]

# 直接評估 traces
results = mlflow.genai.evaluate(
    data=traces,
    scorers=scorers,
)
Code language: PHP (php)

import mlflow
from openai import OpenAI

client = OpenAI()

# Prompt 版本 1：簡潔風格
PROMPT_V1 = [
    {"role": "system", "content": "你是一個助手。簡潔回答問題。"},
    {"role": "user", "content": "問題: {question}"},
]

# Prompt 版本 2：詳細風格
PROMPT_V2 = [
    {"role": "system", "content": "你是一個專業的技術助手。請詳細解釋概念，並提供相關範例。"},
    {"role": "user", "content": "問題: {question}"},
]
Code language: PHP (php)

eval_dataset = [
    {
        "inputs": {"question": "什麼是 Docker？"},
        "expectations": {
            "key_concepts": ["容器", "虛擬化", "映像檔", "隔離"]
        },
    },
    {
        "inputs": {"question": "如何建立 REST API？"},
        "expectations": {
            "key_concepts": ["HTTP", "端點", "請求", "回應"]
        },
    },
    {
        "inputs": {"question": "Git 和 GitHub 有什麼不同？"},
        "expectations": {
            "key_concepts": ["版本控制", "本地", "遠端", "平台"]
        },
    },
]
Code language: JavaScript (javascript)

from mlflow.genai.scorers import RelevanceToQuery, Completeness, Guidelines
from mlflow.genai.judges import make_judge
from typing import Literal

# 自訂 Scorer：概念涵蓋度
concept_coverage = make_judge(
    name="concept_coverage",
    instructions="""
    檢查回答是否涵蓋預期的關鍵概念。

    預期概念: {{ expectations }}
    回答: {{ outputs }}

    評估回答涵蓋了多少預期概念。
    """,
    feedback_value_type=Literal["full", "partial", "minimal"],
    model="openai:/gpt-4o-mini",
)

scorers = [
    RelevanceToQuery(),
    Completeness(),
    Guidelines(name="is_professional", guidelines="回答必須專業且有條理"),
    concept_coverage,
]
Code language: PHP (php)

# 版本 1 的 predict function
@mlflow.trace
def predict_v1(question: str) -> str:
    messages = [
        {"role": "system", "content": "你是一個助手。簡潔回答問題。"},
        {"role": "user", "content": f"問題: {question}"},
    ]
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=messages,
    )
    return response.choices[0].message.content

# 版本 2 的 predict function
@mlflow.trace
def predict_v2(question: str) -> str:
    messages = [
        {"role": "system", "content": "你是一個專業的技術助手。請詳細解釋概念，並提供相關範例。"},
        {"role": "user", "content": f"問題: {question}"},
    ]
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=messages,
    )
    return response.choices[0].message.content

# 分別評估兩個版本
with mlflow.start_run(run_name="prompt_v1_evaluation"):
    results_v1 = mlflow.genai.evaluate(
        data=eval_dataset,
        predict_fn=predict_v1,
        scorers=scorers,
    )

with mlflow.start_run(run_name="prompt_v2_evaluation"):
    results_v2 = mlflow.genai.evaluate(
        data=eval_dataset,
        predict_fn=predict_v2,
        scorers=scorers,
    )
Code language: PHP (php)

# 取得評估結果的 DataFrame
print("=== Prompt V1 結果 ===")
print(results_v1.tables["eval_results"])

print("\n=== Prompt V2 結果 ===")
print(results_v2.tables["eval_results"])
Code language: PHP (php)

設計 Prompt → 追蹤執行 → 評估品質 → 改善 Prompt → ...
    ↑                                    |
    └────────────────────────────────────┘