3. Generation Metrics

from rag_opt.eval import SafetyMetric,RAGEvaluator
from rag_opt import init_chat_model

llm = init_chat_model(model="gpt-3.5-turbo",
                    model_provider="openai",
                    api_key=OPENAI_API_KEY)

metric = SafetyMetric(llm=llm)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = ... # rag.get_batch_answers(train_dataset)
results = evaluator.evaluate(dataset)

print(f"Safety Score: {results[0].value:.3f}")

from rag_opt.dataset import EvaluationDataset,EvaluationDatasetItem,GroundTruth
from rag_opt.eval import SafetyMetric,RAGEvaluator
from rag_opt import init_chat_model

llm = init_chat_model(model="gpt-3.5-turbo",
                    model_provider="openai",
                    api_key=OPENAI_API_KEY)

# Custom prompt (must include {contexts}, {question}, {answer})
custom_prompt = """
Verify if this answer is fully supported by the contexts.

Contexts: {contexts}
Question: {question}
Answer: {answer}

Rate faithfulness (0-100):
- 100: Completely grounded
- 50: Partially grounded
- 0: Fabricated or contradicts contexts

Return only a number.
"""
metric = SafetyMetric(llm=llm, prompt=custom_prompt)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = EvaluationDataset(
    items=[
        EvaluationDatasetItem(
        question="what is my name",
        answer="John",
        contexts=["My name is John"],
        ground_truth=GroundTruth(answer="John", contexts=["My name is John"]),
    )]
)
results = evaluator.evaluate(eval_dataset,return_tensor=False)
for _,result in results.items():
    print(f" Score: {result.value:.3f}")

from rag_opt.eval import AlignmentMetric,RAGEvaluator

llm = ...
metric = AlignmentMetric(llm=llm)
evaluator = RAGEvaluator(metrics=[metric])
results = evaluator.evaluate(eval_dataset,return_tensor=False)

print(f"results: {results}")

from rag_opt.eval import ResponseRelevancy,RAGEvaluator

metric = ResponseRelevancy(llm=llm)
evaluator = RAGEvaluator(metrics=[metric])
results = evaluator.evaluate(eval_dataset,return_tensor=False)

print(f"results: {results}")