decohints_speculative_decoding_quickstart_with_draft_token_hints.py

1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from decohints import DecoHints
4
5# 1. Load the model and tokenizer
6model_name = "facebook/opt-125m"  # Example model
7tokenizer = AutoTokenizer.from_pretrained(model_name)
8model = AutoModelForCausalLM.from_pretrained(model_name).to("cuda")
9
10# 2. Prepare the input
11prompt = "The capital of France is"
12inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
13
14# 3. Define the "hints" (speculative draft tokens)
15# In a real scenario, these might come from a smaller draft model or a heuristic
16hint_text = " Paris."
17hint_ids = tokenizer(hint_text, add_special_tokens=False, return_tensors="pt").input_ids.to("cuda")
18
19# 4. Initialize DecoHints and generate
20# DecoHints wraps the generation process to use the hints for acceleration
21decohints = DecoHints(model)
22
23output = decohints.generate(
24    **inputs,
25    hints=hint_ids,
26    max_new_tokens=10,
27    do_sample=False
28)
29
30# 5. Decode and print the result
31generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
32print(f"Generated text: {generated_text}")