143 lines
4.1 KiB
YAML
143 lines
4.1 KiB
YAML
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: litellm
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: litellm-config-file
|
|
namespace: litellm
|
|
data:
|
|
config.yaml: |
|
|
model_list:
|
|
- model_name: gpt-5.6-luna
|
|
litellm_params:
|
|
model: openai/gpt-5.6-luna
|
|
api_key: "os.environ/OPENAI_API_KEY"
|
|
- model_name: claude-haiku-4.5
|
|
litellm_params:
|
|
model: "anthropic/claude-haiku-4-5-20251001"
|
|
api_key: "os.environ/ANTHROPIC_API_KEY"
|
|
- model_name: claude-sonnet-5
|
|
litellm_params:
|
|
model: "anthropic/claude-sonnet-5"
|
|
api_key: "os.environ/ANTHROPIC_API_KEY"
|
|
- model_name: gemini-3.6-flash
|
|
litellm_params:
|
|
model: gemini/gemini-3.6-flash
|
|
api_key: "os.environ/GEMINI_API_KEY"
|
|
- model_name: tencent/hy3
|
|
litellm_params:
|
|
model: openrouter/tencent/hy3
|
|
api_key: "os.environ/OPENROUTER_API_KEY"
|
|
- model_name: z-ai/glm-5.2
|
|
litellm_params:
|
|
model: openrouter/z-ai/glm-5.2
|
|
api_key: "os.environ/OPENROUTER_API_KEY"
|
|
- model_name: glm-4.7-flash
|
|
litellm_params:
|
|
model: ollama/glm-4.7-flash
|
|
api_base: http://10.88.20.12:11434
|
|
# In-cluster models served by llama.cpp (llama-server, Vulkan on the
|
|
# NUCBox APU). The chat template is applied via --jinja; reasoning (when
|
|
# enabled) comes back in `reasoning_content`.
|
|
- model_name: deepseek-v4-flash-0731
|
|
litellm_params:
|
|
model: openai/deepseek-v4-flash-0731
|
|
api_base: http://llamacpp-deepseek-v4-flash-0731.llamacpp/v1
|
|
api_key: "sk-no-auth"
|
|
- model_name: qwen3.6-27b
|
|
litellm_params:
|
|
model: openai/qwen3.6-27b
|
|
api_base: http://llamacpp-qwen36-27b.llamacpp/v1
|
|
api_key: "sk-no-auth"
|
|
router_settings:
|
|
fallbacks:
|
|
- deepseek-v4-flash-0731: [qwen3.6-27b]
|
|
litellm_settings:
|
|
#set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
|
|
callbacks: ["arize_phoenix"]
|
|
drop_params: True
|
|
# max_budget: 100
|
|
# budget_duration: 30d
|
|
num_retries: 5
|
|
request_timeout: 600
|
|
telemetry: False
|
|
general_settings:
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
|
store_model_in_db: True
|
|
proxy_budget_rescheduler_min_time: 60
|
|
proxy_budget_rescheduler_max_time: 64
|
|
proxy_batch_write_at: 1
|
|
database_connection_pool_limit: 10
|
|
# database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # [OPTIONAL] use for token-based auth to proxy
|
|
environment_variables:
|
|
# settings for using redis caching
|
|
# REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com
|
|
# REDIS_PORT: "16337"
|
|
# REDIS_PASSWORD:
|
|
PHOENIX_COLLECTOR_HTTP_ENDPOINT: "http://phoenix.rogi.casa/v1/traces"
|
|
PHOENIX_PROJECT_NAME: "litellm"
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: litellm-deployment
|
|
namespace: litellm
|
|
labels:
|
|
app: litellm
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: litellm
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: litellm
|
|
spec:
|
|
containers:
|
|
- name: litellm
|
|
image: ghcr.io/berriai/litellm:main-latest # it is recommended to fix a version generally
|
|
args:
|
|
- "--config"
|
|
- "/app/proxy_server_config.yaml"
|
|
ports:
|
|
- containerPort: 4000
|
|
resources:
|
|
requests:
|
|
memory: "512Mi"
|
|
cpu: "500m"
|
|
limits:
|
|
memory: "2Gi"
|
|
cpu: "1000m"
|
|
volumeMounts:
|
|
- name: config-volume
|
|
mountPath: /app/proxy_server_config.yaml
|
|
subPath: config.yaml
|
|
envFrom:
|
|
- secretRef:
|
|
name: litellm-secrets
|
|
env:
|
|
- name: STORE_MODEL_IN_DB
|
|
value: "True"
|
|
volumes:
|
|
- name: config-volume
|
|
configMap:
|
|
name: litellm-config-file
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: litellm-service
|
|
namespace: litellm
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: litellm
|
|
ports:
|
|
- name: http
|
|
port: 80
|
|
targetPort: 4000
|
|
|