anyscale_endpoints_openai_chat_completion_with_llama.py

1import openai
2import os
3
4# To use Anyscale Endpoints, you need to set your API key and the base URL.
5# You can get your API key from the Anyscale Endpoints console.
6client = openai.OpenAI(
7    base_url="https://api.endpoints.anyscale.com/v1",
8    api_key=os.environ.get("ANYSCALE_API_KEY"),
9)
10
11chat_completion = client.chat.completions.create(
12    model="meta-llama/Meta-Llama-3-8B-Instruct",
13    messages=[
14        {"role": "system", "content": "You are a helpful assistant."},
15        {"role": "user", "content": "Say this is a test"},
16    ],
17    temperature=0.7,
18)
19
20print(chat_completion.choices[0].message.content)