120 lines
4.3 KiB
YAML
120 lines
4.3 KiB
YAML
|
# =============================================================================
|
||
|
|
# LiteLLM Proxy Config (DB-backed UI, no Redis)
|
||
|
|
# - Single-node deployment (1 CPU / 2 GB RAM)
|
||
|
|
# - Postgres used for:
|
||
|
|
# - Users / login
|
||
|
|
# - Virtual keys & teams
|
||
|
|
# - Spend tracking (optional)
|
||
|
|
# - No Redis caching (cache: false)
|
||
|
|
#
|
||
|
|
# Docs:
|
||
|
|
# https://docs.litellm.ai/docs/proxy/configs
|
||
|
|
# https://docs.litellm.ai/docs/proxy/config_settings
|
||
|
|
# https://docs.litellm.ai/docs/proxy/virtual_keys
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
# Models exposed by the proxy
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
model_list:
|
||
|
|
# --- OpenAI examples -------------------------------------------------------
|
||
|
|
#- model_name: gpt-4o
|
||
|
|
# litellm_params:
|
||
|
|
# model: openai/gpt-4o
|
||
|
|
# api_key: os.environ/OPENAI_API_KEY
|
||
|
|
|
||
|
|
#- model_name: gpt-4o-mini
|
||
|
|
# litellm_params:
|
||
|
|
# model: openai/gpt-4o-mini
|
||
|
|
# api_key: os.environ/OPENAI_API_KEY
|
||
|
|
|
||
|
|
# --- Anthropic example -----------------------------------------------------
|
||
|
|
# - model_name: claude-3-5-sonnet
|
||
|
|
# litellm_params:
|
||
|
|
# model: anthropic/claude-3-5-sonnet-20241022
|
||
|
|
# api_key: os.environ/ANTHROPIC_API_KEY
|
||
|
|
|
||
|
|
# --- Groq example ----------------------------------------------------------
|
||
|
|
# - model_name: groq-llama-3.3-70b
|
||
|
|
# litellm_params:
|
||
|
|
# model: groq/llama-3.3-70b-versatile
|
||
|
|
# api_key: os.environ/GROQ_API_KEY
|
||
|
|
|
||
|
|
# --- Azure OpenAI example --------------------------------------------------
|
||
|
|
# - model_name: azure-gpt-4o
|
||
|
|
# litellm_params:
|
||
|
|
# model: azure/my_azure_deployment
|
||
|
|
# api_base: os.environ/AZURE_API_BASE
|
||
|
|
# api_key: os.environ/AZURE_API_KEY
|
||
|
|
# api_version: "2025-01-01-preview"
|
||
|
|
|
||
|
|
# --- Local Ollama example --------------------------------------------------
|
||
|
|
# - model_name: ollama-llama3
|
||
|
|
# litellm_params:
|
||
|
|
# model: ollama/llama3
|
||
|
|
# api_base: http://host.docker.internal:11434
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
# Core LiteLLM behavior (no Redis cache)
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
litellm_settings:
|
||
|
|
# Retries & timeouts
|
||
|
|
num_retries: 2
|
||
|
|
request_timeout: 60 # seconds
|
||
|
|
|
||
|
|
# Disable caching entirely to avoid Redis
|
||
|
|
cache: false # <- IMPORTANT: no Redis in your stack
|
||
|
|
|
||
|
|
# Drop unsupported provider-specific params instead of erroring
|
||
|
|
drop_params: true
|
||
|
|
|
||
|
|
# If you later enable caching, add cache_params here with Redis config.
|
||
|
|
# cache_params:
|
||
|
|
# type: redis
|
||
|
|
# host: litellm-redis
|
||
|
|
# port: 6379
|
||
|
|
# password: os.environ/REDIS_PASSWORD
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
# Router settings (all in-memory, single-node)
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
router_settings:
|
||
|
|
routing_strategy: "simple-shuffle" # simple-shuffle | least-busy | latency-based-routing
|
||
|
|
num_retries: 1
|
||
|
|
timeout: 30
|
||
|
|
retry_after: 5
|
||
|
|
|
||
|
|
# No Redis routing / transaction buffer on this small single-node setup
|
||
|
|
# redis_host: litellm-redis
|
||
|
|
# redis_port: 6379
|
||
|
|
# redis_password: os.environ/REDIS_PASSWORD
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
# General settings (DB-backed UI, no Redis)
|
||
|
|
# -----------------------------------------------------------------------------#
|
||
|
|
general_settings:
|
||
|
|
# Admin master key (used for UI + admin API).
|
||
|
|
# Must match LITELLM_MASTER_KEY from .env and start with "sk-".
|
||
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
||
|
|
|
||
|
|
# Postgres connection (from DATABASE_URL env var).
|
||
|
|
# Required for:
|
||
|
|
# - /ui login and user management
|
||
|
|
# - virtual keys
|
||
|
|
# - spend tracking
|
||
|
|
database_url: os.environ/DATABASE_URL
|
||
|
|
|
||
|
|
# Store model definitions in DB (used by the UI)
|
||
|
|
store_model_in_db: true
|
||
|
|
|
||
|
|
# Telemetry & metrics
|
||
|
|
telemetry: false
|
||
|
|
enable_prometheus: true
|
||
|
|
|
||
|
|
# On a small box, you may want to reduce some DB-heavy features.
|
||
|
|
# You can still track spend, but disable some advanced cost mgmt.
|
||
|
|
disable_end_user_cost_tracking: false
|
||
|
|
disable_reset_budget: false
|
||
|
|
|
||
|
|
# No Redis transaction buffer here
|
||
|
|
# use_redis_transaction_buffer: false
|