lm_eval_huggingface_model_evaluation_with_simple_evaluate.py

1import lm_eval
2from lm_eval.models.huggingface import HFLM
3
4# 1. Initialize the model (e.g., a Hugging Face model)
5# You can also pass a string like "hf" to simple_evaluate directly
6model = HFLM(pretrained="gpt2")
7
8# 2. Run the evaluation
9results = lm_eval.simple_evaluate(
10    model=model,
11    tasks=["hellaswag", "arc_easy"],
12    num_fewshot=0,
13    batch_size=16,
14    device="cuda:0" # or "cpu"
15)
16
17# 3. Print the results
18print(results["results"])