apiVersion: v1
kind: Namespace
metadata:
  name: litellm
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: litellm-config-file
  namespace: litellm
data:
  config.yaml: |
    model_list:
      - model_name: gpt-5.6-luna
        litellm_params:
          model: openai/gpt-5.6-luna
          api_key: "os.environ/OPENAI_API_KEY"
      - model_name: claude-haiku-4.5
        litellm_params:
          model: "anthropic/claude-haiku-4-5-20251001"
          api_key: "os.environ/ANTHROPIC_API_KEY"
      - model_name: claude-sonnet-5
        litellm_params:
          model: "anthropic/claude-sonnet-5"
          api_key: "os.environ/ANTHROPIC_API_KEY"
      - model_name: gemini-3.6-flash
        litellm_params:
          model: gemini/gemini-3.6-flash
          api_key: "os.environ/GEMINI_API_KEY"
      - model_name: tencent/hy3
        litellm_params:
          model: openrouter/tencent/hy3
          api_key: "os.environ/OPENROUTER_API_KEY"
      - model_name: z-ai/glm-5.2
        litellm_params:
          model: openrouter/z-ai/glm-5.2
          api_key: "os.environ/OPENROUTER_API_KEY"
      - model_name: glm-4.7-flash
        litellm_params:
          model: ollama/glm-4.7-flash
          api_base: http://10.88.20.12:11434
      # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer).
      # Served by llama.cpp (llama-server, Vulkan on the NUCBox APU) — see
      # llamacpp/deployment-qwen36-27b.yaml. Thinking is left ON; reasoning comes
      # back in `reasoning_content` (llama-server --jinja + default reasoning format).
      - model_name: qwen3.6-27b
        litellm_params:
          model: openai/qwen3.6-27b
          api_base: http://llamacpp-qwen36-27b.llamacpp/v1
          api_key: "sk-no-auth"
      # Faster "flash" variant: Qwen3.6-35B-A3B is a MoE (3B active params), so
      # inference is much faster than the dense 27B despite more total weights.
      # See llamacpp/deployment-qwen36-35b-a3b.yaml.
      - model_name: qwen3.6-35b-a3b
        litellm_params:
          model: openai/qwen3.6-35b-a3b
          api_base: http://llamacpp-qwen36-35b-a3b.llamacpp/v1
          api_key: "sk-no-auth"
    litellm_settings:
      #set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production
      callbacks: ["arize_phoenix"]
      drop_params: True
      # max_budget: 100 
      # budget_duration: 30d
      num_retries: 5
      request_timeout: 600
      telemetry: False
    general_settings: 
      master_key: os.environ/LITELLM_MASTER_KEY
      store_model_in_db: True
      proxy_budget_rescheduler_min_time: 60
      proxy_budget_rescheduler_max_time: 64
      proxy_batch_write_at: 1
      database_connection_pool_limit: 10
      # database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # [OPTIONAL] use for token-based auth to proxy
    environment_variables:
      # settings for using redis caching
      # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com
      # REDIS_PORT: "16337"
      # REDIS_PASSWORD: 
      PHOENIX_COLLECTOR_HTTP_ENDPOINT: "http://phoenix.rogi.casa/v1/traces"
      PHOENIX_PROJECT_NAME: "litellm"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: litellm-deployment
  namespace: litellm
  labels:
    app: litellm
spec:
  selector:
    matchLabels:
      app: litellm
  template:
    metadata:
      labels:
        app: litellm
    spec:
      containers:
      - name: litellm
        image: ghcr.io/berriai/litellm:main-latest # it is recommended to fix a version generally
        args:
          - "--config"
          - "/app/proxy_server_config.yaml"
        ports:
        - containerPort: 4000
        resources:
          requests:
            memory: "512Mi"
            cpu: "500m"
          limits:
            memory: "2Gi"
            cpu: "1000m"
        volumeMounts:
        - name: config-volume
          mountPath: /app/proxy_server_config.yaml
          subPath: config.yaml
        envFrom:
        - secretRef:
            name: litellm-secrets
        env:
        - name: STORE_MODEL_IN_DB
          value: "True"
      volumes:
        - name: config-volume
          configMap:
            name: litellm-config-file
---
apiVersion: v1
kind: Service
metadata:
  name: litellm-service
  namespace: litellm
spec:
  type: ClusterIP
  selector:
    app: litellm
  ports:
    - name: http
      port: 80
      targetPort: 4000