anyscale_endpoints_openai_compatible_chat_completion_quickstart.py

1import os
2from openai import OpenAI
3
4# Initialize the client with Anyscale Endpoints base URL and your API key
5# Ensure the ANYSCALE_API_KEY environment variable is set
6client = OpenAI(
7    base_url="https://api.endpoints.anyscale.com/v1",
8    api_key=os.environ.get("ANYSCALE_API_KEY"),
9)
10
11# Create a chat completion request
12chat_completion = client.chat.completions.create(
13    model="meta-llama/Meta-Llama-3-8B-Instruct",
14    messages=[
15        {"role": "system", "content": "You are a helpful assistant."},
16        {"role": "user", "content": "What are the benefits of using Anyscale for LLM serving?"}
17    ],
18    temperature=0.7
19)
20
21# Print the model's response
22print(chat_completion.choices[0].message.content)