2. Retrieval Metrics

from rag_opt.eval import RAGEvaluator,ContextPrecision
from rag_opt import init_chat_model

llm = init_chat_model(model="gpt-3.5-turbo",
                    model_provider="openai",
                    api_key=OPENAI_API_KEY)

metric = ContextPrecision(llm=llm, limit_contexts=5)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = ... # evaluation dataset from RAG
results = evaluator.evaluate(eval_dataset,return_tensor=False)

from rag_opt.eval import RAGEvaluator,ContextRecall
from rag_opt import init_chat_model

llm = init_chat_model(model="gpt-3.5-turbo",
                    model_provider="openai",
                    api_key=OPENAI_API_KEY)

metric = ContextRecall(llm=llm)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = ... # rag.get_batch_answers(train_dataset)
results = evaluator.evaluate(eval_dataset,return_tensor=False)

from rag_opt.eval import RAGEvaluator,MRR
from rag_opt import init_embeddings

# embedding is required for MRR
embeddings = init_embeddings(model="all-MiniLM-L6-v2",
                             model_provider="huggingface",
                             api_key=HUGGINFACE_API_KEY)

metric = MRR(embedding_model=embeddings)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = ... # rag.get_batch_answers(train_dataset)
results = evaluator.evaluate(eval_dataset,return_tensor=False)

from rag_opt.eval import RAGEvaluator,NDCG
from rag_opt import init_embeddings

# embedding is required for NDCG
embeddings = init_embeddings(model="all-MiniLM-L6-v2",
                             model_provider="huggingface",
                             api_key=HUGGINFACE_API_KEY)

metric = NDCG(embedding_model=embeddings)
evaluator = RAGEvaluator(metrics=[metric])
eval_dataset = ... # rag.get_batch_answers(train_dataset)
results = evaluator.evaluate(eval_dataset,return_tensor=False)

rom rag_opt.eval import ContextPrecision, ContextRecall, MRR, NDCG
from rag_opt import init_chat_model, init_embeddings


llm = init_chat_model(model="gpt-3.5-turbo",
                    model_provider="openai",
                    api_key=OPENAI_API_KEY)

embeddings = init_embeddings(model="all-MiniLM-L6-v2",
                             model_provider="huggingface",
                             api_key=HUGGINFACE_API_KEY)

metrics = [
    ContextPrecision(llm=llm, limit_contexts=5),
    ContextRecall(llm=llm),
    MRR(embedding_model=embeddings),
    NDCG(embedding_model=embeddings)
]

evaluator = RAGEvaluator(metrics=metrics)
eval_dataset = ... # rag.get_batch_answers(train_dataset)
results = evaluator.evaluate(eval_dataset,return_tensor=False)

for result in results.items():
    print(f"{result.name}: {result.value:.3f}")