LLM-as-a-Judge: Evaluate Your Models Without Human Reviewers

from openai import OpenAI
from pydantic import BaseModel

client = OpenAI()

class EvalResult(BaseModel):
    score: int
    reasoning: str

def judge_output(
    question: str,
    answer: str,
    criteria: str = "relevance and accuracy",
) -> EvalResult:
    """Use an LLM to evaluate another LLM's output."""
    response = client.beta.chat.completions.parse(
        model="gpt-4o",
        response_format=EvalResult,
        messages=[
            {
                "role": "system",
                "content": (
                    "You are an expert evaluator. Score the answer "
                    "on a scale of 1-10 based on the given criteria. "
                    "Provide chain-of-thought reasoning before scoring."
                ),
            },
            {
                "role": "user",
                "content": (
                    f"Criteria: {criteria}\n\n"
                    f"Question: {question}\n\n"
                    f"Answer: {answer}\n\n"
                    "Evaluate this answer. Return your reasoning "
                    "and a score from 1-10."
                ),
            },
        ],
    )
    return response.choices[0].message.parsed

result = judge_output(
    question="What causes a Python deadlock?",
    answer="A deadlock occurs when two threads each hold a lock the other needs.",
    criteria="technical accuracy and completeness",
)
print(f"Score: {result.score}/10")
print(f"Reasoning: {result.reasoning}")

pip install -U deepeval

export OPENAI_API_KEY="your_api_key"

from deepeval.test_case import LLMTestCase, LLMTestCaseParams
from deepeval.metrics import GEval

coherence_metric = GEval(
    name="Coherence",
    criteria=(
        "Coherence - the collective quality of all sentences "
        "in the actual output. Sentences should flow logically, "
        "maintain consistent terminology, and build on each other."
    ),
    evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT],
    threshold=0.7,
)

test_case = LLMTestCase(
    input="Explain gradient descent in simple terms.",
    actual_output=(
        "Gradient descent is an optimization algorithm. "
        "It finds the minimum of a function by iteratively "
        "moving in the direction of steepest descent. "
        "Think of it as a ball rolling downhill — it naturally "
        "settles at the lowest point."
    ),
)

coherence_metric.measure(test_case)
print(f"Score: {coherence_metric.score}")
print(f"Reason: {coherence_metric.reason}")

from deepeval import evaluate
from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric

relevancy = AnswerRelevancyMetric(threshold=0.7)
faithfulness = FaithfulnessMetric(threshold=0.7)

test_case = LLMTestCase(
    input="What are the side effects of gradient clipping?",
    actual_output=(
        "Gradient clipping prevents exploding gradients by capping "
        "the gradient norm. Side effects include slower convergence "
        "when the clip threshold is too aggressive, and potential "
        "loss of gradient direction information."
    ),
    retrieval_context=[
        "Gradient clipping caps gradient norms to prevent exploding "
        "gradients. Setting the threshold too low can slow convergence. "
        "Clipping by norm preserves direction better than clipping by value."
    ],
)

results = evaluate(
    test_cases=[test_case],
    metrics=[relevancy, faithfulness, coherence_metric],
)

from openai import OpenAI
from pydantic import BaseModel, Field
from enum import Enum

client = OpenAI()

class Winner(str, Enum):
    A = "A"
    B = "B"
    TIE = "TIE"

class PairwiseResult(BaseModel):
    winner: Winner
    reasoning: str
    confidence: float = Field(ge=0.0, le=1.0)

def compare_outputs(
    question: str,
    output_a: str,
    output_b: str,
    criteria: str = "accuracy, completeness, and clarity",
) -> PairwiseResult:
    """Compare two LLM outputs and pick the better one."""
    response = client.beta.chat.completions.parse(
        model="gpt-4o",
        response_format=PairwiseResult,
        messages=[
            {
                "role": "system",
                "content": (
                    "You are an expert evaluator comparing two answers. "
                    "Evaluate based on the given criteria. Be specific "
                    "about WHY one answer is better. If both are equally "
                    "good, say TIE."
                ),
            },
            {
                "role": "user",
                "content": (
                    f"Criteria: {criteria}\n\n"
                    f"Question: {question}\n\n"
                    f"Answer A: {output_a}\n\n"
                    f"Answer B: {output_b}\n\n"
                    "Which answer is better? Return the winner, "
                    "reasoning, and your confidence level (0-1)."
                ),
            },
        ],
    )
    return response.choices[0].message.parsed

result = compare_outputs(
    question="How does backpropagation work?",
    output_a="Backpropagation computes gradients using the chain rule.",
    output_b=(
        "Backpropagation computes gradients of the loss function "
        "with respect to each weight by applying the chain rule "
        "backwards through the network layers. Each layer's gradient "
        "depends on the gradient of the layer above it, propagated "
        "through the activation function's derivative."
    ),
    criteria="technical depth and educational value",
)
print(f"Winner: {result.winner}")
print(f"Confidence: {result.confidence}")
print(f"Why: {result.reasoning}")

def compare_with_debiasing(
    question: str,
    output_a: str,
    output_b: str,
    criteria: str = "accuracy, completeness, and clarity",
) -> PairwiseResult:
    """Run pairwise comparison twice with swapped order."""
    result_ab = compare_outputs(question, output_a, output_b, criteria)
    result_ba = compare_outputs(question, output_b, output_a, criteria)

    # If both agree on the same winner, the result is reliable
    if result_ab.winner == Winner.A and result_ba.winner == Winner.B:
        return result_ab  # Both say output_a is better
    if result_ab.winner == Winner.B and result_ba.winner == Winner.A:
        return result_ab  # Both say output_b is better

    # Disagreement — call it a tie
    return PairwiseResult(
        winner=Winner.TIE,
        reasoning="Position bias detected: results flipped with order.",
        confidence=0.5,
    )