sglang_runtime_local_server_offline_inference_quickstart.py

1import sglang as sgl
2
3def multi_chain_generate():
4    state = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf")
5    
6    @sgl.function
7    def simple_qa(s, question):
8        s += "Q: " + question + "\n"
9        s += "A: " + sgl.gen("answer", stop="\n")
10
11    state = simple_qa.run(question="What is the capital of France?")
12    print(state["answer"])
13
14if __name__ == "__main__":
15    multi_chain_generate()