Llama 3.3 70B quantized to fp8 precision, optimized to be faster.
Llama-3.3-70b-Instruct-Fp8-Fast (meta/llama-3.3-70b-instruct-fp8-fast) is a text model from Meta, released 2024-12-06. Context window: 131,072 tokens; max output 8,192. Pricing via AIgateway: input $0.290/M tokens, output $2.25/M tokens. Capabilities: tools, streaming, json, async. Call it via https://api.aigateway.sh/v1/chat/completions with the OpenAI SDK — set model="meta/llama-3.3-70b-instruct-fp8-fast". Best for: Chat, RAG, Classification.
curl https://api.aigateway.sh/v1/chat/completions \
-H "Authorization: Bearer $AIGATEWAY_API_KEY" \
-H "Content-Type: application/json" \
-d '{"model":"meta/llama-3.3-70b-instruct-fp8-fast","messages":[{"role":"user","content":"hello"}],"stream":true}'{
"model": "meta/llama-3.3-70b-instruct-fp8-fast",
"messages": [
{ "role": "system", "content": "You are a helpful assistant." },
{ "role": "user", "content": "Hello!" }
],
"temperature": 0.7,
"top_p": 0.95,
"max_tokens": 1024,
"stream": false,
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather for a city",
"parameters": {
"type": "object",
"properties": {
"city": { "type": "string" }
},
"required": ["city"]
}
}
}
],
"tool_choice": "auto",
"parallel_tool_calls": true,
"response_format": { "type": "json_object" }
}{
"id": "chatcmpl-abc123",
"object": "chat.completion",
"created": 1776947082,
"model": "meta/llama-3.3-70b-instruct-fp8-fast",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hello! How can I help you today?",
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"Tokyo\"}"
}
}
]
},
"finish_reason": "stop" // or "tool_calls" when calling a function
}
],
"usage": {
"prompt_tokens": 24,
"completion_tokens": 12,
"total_tokens": 36
}
}"stream": true// 1. Role announcement (first chunk):
data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
// 2. Content chunks (final answer):
data: {"choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}
data: {"choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}
// Tool-call chunks (when the assistant calls a function):
data: {"choices":[{"index":0,"delta":{"tool_calls":[
{"index":0,"id":"call_abc","type":"function",
"function":{"name":"get_weather","arguments":""}}]},"finish_reason":null}]}
data: {"choices":[{"index":0,"delta":{"tool_calls":[
{"index":0,"function":{"arguments":"{\"city\":"}}]},"finish_reason":null}]}
data: {"choices":[{"index":0,"delta":{"tool_calls":[
{"index":0,"function":{"arguments":"\"Tokyo\"}"}}]},"finish_reason":null}]}
// Concat arguments fragments by index → {"city":"Tokyo"}
// Finish chunk:
data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
// Terminator:
data: [DONE]# pip install aigateway-py openai
# aigateway-py adds sub-accounts, evals, replays, jobs, webhook verify.
# openai SDK covers chat — drop-in per our SDK's own guidance.
from openai import OpenAI
client = OpenAI(
base_url="https://api.aigateway.sh/v1",
api_key="sk-aig-...",
)
stream = client.chat.completions.create(
model="meta/llama-3.3-70b-instruct-fp8-fast",
messages=[{"role": "user", "content": "Hello!"}],
stream=True,
)
for chunk in stream:
print(chunk.choices[0].delta.content or "", end="", flush=True)