Configuration reference

# add this modeline for validation in vscode
# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
#
# llama-swap YAML configuration example
# -------------------------------------
#
# 💡 Tip - Use an LLM with this file!
# ====================================
#  This example configuration is written to be LLM friendly. Try
#  copying this file into an LLM and asking it to explain or generate
#  sections for you.
# ====================================

# Usage notes:
# - Below are all the available configuration options for llama-swap.
# - Settings noted as "required" must be in your configuration file
# - Settings noted as "optional" can be omitted

# healthCheckTimeout: number of seconds to wait for a model to be ready to serve requests
# - optional, default: 120
# - minimum value is 15 seconds, anything less will be set to this value
# - when the health check reports loading progress, the countdown restarts each
#   time the progress value changes, so this limits how long a load may go
#   without progress
healthCheckTimeout: 500

# logLevel: sets the logging value
# - optional, default: info
# - Valid log levels: debug, info, warn, error
logLevel: info

# logTimeFormat: enables and sets the logging timestamp format
# - optional, default (disabled): ""
# - Valid values: "", "ansic", "unixdate", "rubydate", "rfc822", "rfc822z",
#   "rfc850", "rfc1123", "rfc1123z", "rfc3339", "rfc3339nano", "kitchen",
#   "stamp", "stampmilli", "stampmicro", and "stampnano".
# - For more info, read: https://pkg.go.dev/time#pkg-constants
logTimeFormat: ""

# logToStdout: controls which log streams are written to stdout
# - optional, default: "proxy"
# - a comma separated list of one or more streams, e.g. "proxy,http":
#   - "proxy": logs generated by llama-swap when swapping models,
#      handling requests, etc.
#   - "upstream": a copy of an upstream processes stdout logs
#   - "http": one access log line per HTTP request
# - or one of these values:
#   - "both": every stream (proxy, upstream and http) interleaved together
#   - "none": no logs are ever written to stdout
# - HTTP access lines used to be part of the proxy logs. Use "proxy,http" to
#   keep printing them to stdout.
logToStdout: "proxy"

# store: persistent storage for activity metrics state
# - optional, default: in-memory sqlite database capped by metricsMaxInMemory
# - path is a sqlite database file path
# - file-backed sqlite keeps activity logs across restarts
# - persistCaptures: keep request/response captures in a second sqlite file
#   next to path, named with _captures added to the base name, so
#   /path/to/file.sqlite stores captures in /path/to/file_captures.sqlite
#   - optional, default: false (captures are kept in memory)
#   - total size is capped by captureBuffer; the oldest captures are deleted
#   - keep or move the two files together
# store:
#   path: /path/to/file.sqlite
#   persistCaptures: true

# metricsMaxInMemory: maximum number of metrics to keep in memory
# - optional, default: 1000
# - controls how many metrics are stored in memory before older ones are discarded
# - useful for limiting memory usage when processing large volumes of metrics
# - when persistent sqlite storage is defined in store.path this has no effect
metricsMaxInMemory: 1000

# captureBuffer: how many MBs of compressed request/response captures to keep
# - optional, default: 5
# - set to 0 to disable
# - also caps the captures database file when store.persistCaptures is true
captureBuffer: 15

# ui.activity.session_id: ordered request headers used to identify sessions in
# the Activity page's in-flight request table.
# - optional, default: ["X-Session-ID", "X-Litellm-Session-Id"]
# - matching is case-insensitive; the first non-empty matching header is shown
# - set to [] to disable session ID lookup
ui:
  activity:
    session_id: ["X-Session-ID", "X-Litellm-Session-Id"]

# performance: configuration for system monitoring statistics
# - timing values are duration strings like 1s, 1h30m, 90m, 2h10s, etc.
performance:
  # disabled: boolean
  # - default: false
  disabled: false

  # every: delay between polling for new performance statistics
  # - default: 5s
  # - minimum duration 5s
  every: 15s

# helpagent: the UI's Help page, a chat that answers questions about llama-swap
# from its documentation, configuration and source code
helpagent:
  # disable: boolean
  # - optional, default: false
  # - true hides Help in the UI and does not serve the /api/mcp endpoint
  #   whose tools the Help page uses
  disable: false

# startPort: sets the starting port number for the automatic ${PORT} macro.
# - optional, default: 5800
# - the ${PORT} macro can be used in model.cmd, and elsewhere in that model once
#   model.cmd uses it
# - it is automatically incremented for every model that uses it
startPort: 10001

# sendLoadingState: inject loading status updates into the reasoning (thinking)
# field
# - optional, default: false
# - when true, a stream of loading messages will be sent to the client in the
#   reasoning field so chat UIs can show that loading is in progress.
# - see #366 for more details
# - this is for chat clients. To show a real progress bar on the Models page,
#   have your server report progress from its checkEndpoint: see the "Loading
#   progress" section of guides/model-runtime/writing-cmd.
sendLoadingState: true

# includeAliasesInList: present aliases within the /v1/models OpenAI API listing
# - optional, default: false
# - when true, model aliases will be output to the API model listing duplicating
#   all fields except for Id so chat UIs can use the alias equivalent to the original.
includeAliasesInList: false

# globalTTL: the default TTL in seconds before unloading a model
# - optional, default: 0 (never automatically unload)
# - must be >= 0
globalTTL: 0

# unloadTimeout: graceful timeout in seconds when unloading a model (manual, API, or ttl expiry)
# - optional, default: 10
# - used before force-killing the model process on unload
# - can be overridden per model with model.unloadTimeout
# - large values extend how long an unload can take when a process does not exit cleanly
unloadTimeout: 10

# globalConcurrencyLimit: caps the number of inference requests served at once
# across all models combined
# - optional, default: 0 (no limit)
# - unlike models.*.concurrencyLimit, this is a single semaphore shared by
#   every model rather than a per-model limit
# - requests that exceed the limit receive an HTTP 429 Too Many Requests
#   response immediately, rather than being queued
# - see issue #1086
globalConcurrencyLimit: 0

# maxRequestBodyMB: maximum size, in megabytes, of a request body sent to a
# model endpoint (chat, completions, embeddings, audio, images, etc.)
# - optional, default: 250
# - must be 1 or more
# - larger requests receive an HTTP 413 Request Entity Too Large response
# - raise it for large audio uploads or many base64 images in one request
maxRequestBodyMB: 250

# macros: a dictionary of string substitutions
# - optional, default: empty dictionary
# - macros are reusable snippets
# - global macros can be used in any configuration value
# - model macros can be used in any value within that model
# - useful for reducing common configuration settings
# - macro names are strings and must be less than 64 characters
# - macro names must match the regex ^[a-zA-Z0-9_-]+$
# - macro names must not be a reserved name: PID, PORT, or MODEL_ID
# - macro values can be numbers, bools, or strings
# - macros can contain other macros, but they must be defined before they are used
# - environment variables can be referenced with ${env.VAR_NAME} syntax
#   - env macros are substituted first, before regular macros
#   - if the env var is not set, config loading will fail with an error
# - ${PORT} is assigned while loading each model that uses it in cmd
# - ${PID} is substituted when a model's cmdStop runs
macros:
  # Example of a multi-line macro
  "latest-llama": >
    /path/to/llama-server/llama-server-ec9e0301 --port ${PORT}

  "default_ctx": 4096

  # Example of macro-in-macro usage. macros can contain other macros
  # but they must be previously declared.
  "default_args": "--ctx-size ${default_ctx}"

  # Example of environment variable macros
  # - ${env.VAR_NAME} pulls the value from the system environment
  # - useful for paths, secrets, or machine-specific configuration
  "models_dir": "${env.HOME}/models"

# apiKeys: require an API key when making requests to inference endpoints
# - optional, default: []
# - when empty (the default) authorization will not be checked as llama-swap is default-allow
# - each key is a non-empty string
apiKeys:
  - "sk-hunter2"
  # tip, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
  - "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"

  # use environment variable macros to keep secrets out of the config
  - "${env.API_KEY_1}"
  - "${env.API_KEY_2}"

# security: settings that control who may talk to llama-swap
# - optional, omit the whole block to allow any origin (the legacy default)
# - llama-swap only sends Access-Control-* headers when the request carries an
#   Origin header, so curl, litellm and other non-browser clients get none
# - any CORS headers an upstream (llama-server, vLLM) sets are stripped from
#   proxied responses so exactly one set reaches the client, see issue #85
security:
  # cors: allowedOrigins is what switches modes
  # - leave allowedOrigins out to keep the permissive default
  # - set it to take charge of access; nothing then falls back to allow-all
  # - the other settings only describe preflight mechanics, so each one you
  #   leave out still takes its default and naming only origins keeps working
  cors:
    # allowedOrigins: browser origins allowed to read responses
    # - omit the key to allow any origin; an empty list [] is an error
    # - REQUIRED once any other cors setting is set
    # - each entry is a bare scheme://host[:port], no trailing slash or path
    # - "*" allows any origin, and cannot be combined with allowCredentials
    allowedOrigins:
      - "*"

    # allowCredentials: let browsers send cookies and Authorization headers
    # - optional, default: false
    # - requires allowedOrigins to list origins explicitly
    allowCredentials: false

    # allowPrivateNetwork: answer Chrome's Private Network Access preflight,
    #   so a page on a public origin may reach llama-swap on a LAN address
    # - optional, default: false
    # - requires allowedOrigins to list origins explicitly, not "*"
    # - only sent on a preflight that asked for it
    allowPrivateNetwork: false

    # allowedMethods: methods advertised on preflight (OPTIONS) responses
    # - optional, default: [GET, POST, PUT, PATCH, DELETE, OPTIONS]
    allowedMethods:
      - "GET"
      - "POST"
      - "PUT"
      - "PATCH"
      - "DELETE"
      - "OPTIONS"

    # allowedHeaders: request headers advertised on preflight responses
    # - optional, default: [] which echoes the client's
    #   Access-Control-Request-Headers, falling back to
    #   [Content-Type, Authorization, Accept, X-Requested-With]
    allowedHeaders: []

    # exposedHeaders: response headers a browser may read beyond the
    #   CORS-safelisted set
    # - optional, default: [] which omits the header
    exposedHeaders: []

    # maxAge: how long in seconds a browser may cache a preflight response
    # - optional, default: 86400
    maxAge: 86400

# tailcat: configure models exposed over Tailcat virtual TCP port 80
# - requires -listen-tailcat /path/to/server.private.json to start the listener
# - when -listen-tailcat is set, models must contain at least one public ID
# - allow denies by default: clients whose node key is not listed get HTTP 403
# - an empty allow denies every client; "*" permits any client holding the
#   case-sensitive connection token
# - "*" exposes every callable public ID
# - admin defaults false and keeps the UI, /api, logs, metrics and operations private
# - debug defaults false; set true to include verbose Tailcat transport logs in proxy logs
# - allow, models, admin and debug all update on config reload without
#   restarting the listener; changing the -listen-tailcat key requires a restart
tailcat:
  allow:
    - nodekey:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
  models:
    - model_a
    - llama-swap-peer/model_a
  admin: false
  debug: false

# upstream: controls behaviour of the /upstream passthrough endpoint
# - optional, default: empty dictionary
# - recommended to only use in special use cases. Leaving it as the
#   default will typically be the best experience
upstream:
  # ignorePaths: list of RE2 compatible regular expressions
  # - default: (see below)
  # - any request to a path matching any of the regular expressions
  #   will be ignored and not trigger a swap
  ignorePaths:
    - '.*\.(js|json|css|png|gif|jpg|jpeg|ico|txt)$'

    # ComfyUI specific: ignore /ws and /api/jobs to prevent swaps
    # when the UI is open. The /comfyui endpoint applies these on its own;
    # they are only needed to serve ComfyUI through /upstream/<model>/
    - ^\/ws(\/|$)
    - ^\/api\/jobs$

# profiles: named model ID replacements switched at runtime through the UI or API
# - optional, default: empty dictionary
# - one profile or none is active; hooks.on_startup.profile selects the profile
#   used on startup and after a configuration reload
# - pins are applied before aliases, filters, and routing
# - targets may be a local model, fully qualified peer model, alias, setParamsByID alias, or selector
# - an empty string or YAML null (~) disables the pin with a 404; it is not
#   added to model listings, while existing local, alias, or peer IDs remain
profiles:
  "coding":
    description: "Coding-focused model routing"
    pins:
      "llm-code": "gpt-oss-120b"
      "llm-plan": "qwen-unlisted"
      "image-gen": ~

# selectors: virtual model IDs resolved to concrete targets per request
# - optional, default: empty dictionary
# - profiles run first, so a profile pin may target a selector
# - selector IDs cannot collide with model IDs, aliases, or fully qualified peer model names
# - selector targets cannot be other selectors
# - selectors are not supported on /upstream/<model> paths
selectors:
  # warm chooses the first ready target in order, then an already-starting
  # target, and cold-starts the first target when none are running
  "coding-model":
    strategy: warm
    targets:
      - "gpt-oss-120b"
      - "qwen-unlisted"
    name: "Coding Model"
    description: "Best currently loaded coding model"
    metadata:
      purpose: coding

  # pin always uses the first target; remaining entries are reference data
  "llm-plan":
    strategy: pin
    targets:
      - "gpt-oss-120b"
      - "z-ai/glm-4.7"

  # spillover fills local or remote targets to the reservation count in order,
  # starting the next target when the active targets reach that count
  "llama":
    strategy: spillover
    targets:
      - "docker-llama"
      - "modelA"
      - "modelB"
    settings:
      # requests reserved per active target before spilling over to the next target
      # - optional, default: 1
      spillover: 4

# models: a dictionary of model configurations
# - required
# - each key is the model's ID, used in API requests
# - model settings have default values that are used if they are not defined here
# - the model's ID is available in the ${MODEL_ID} macro, also available in macros defined above
# - below are examples of the all the settings a model can have
models:
  # keys are the model names used in API requests
  "gpt-oss-120b":
    # macros: a dictionary of string substitutions specific to this model
    # - optional, default: empty dictionary
    # - macros defined here override macros defined in the global macros section
    # - model level macros follow the same rules as global macros
    macros:
      "default_ctx": 16384
      "temp": 0.7

    # cmd: the command to run to start the inference server.
    # - required
    # - it is just a string, similar to what you would run on the CLI
    # - using `|` allows for comments in the command, these will be parsed out
    # - macros can be used within cmd
    cmd: |
      # ${latest-llama} is a macro that is defined above
      ${latest-llama}
      --model path/to/gpt-oss-120B.gguf
      --ctx-size ${default_ctx}
      --temperature ${temp}

    # capacities: resource requirements for use: capacity
    # - optional, default: empty dictionary (no declared usage)
    # - keys name routing.router.settings.capacity.pools; omitted pools use zero
    # - values: nonnegative integers in the corresponding pool's unit
    # capacities:
    #   ram: 18000
    #   gpu0: 12000

    # evict_cost: manual eviction preference for the capacity router
    # - optional, default: 0; nonnegative integer; higher favors keeping resident
    # evict_cost: 10

    # name: a display name for the model
    # - optional, default: empty string
    # - if set, it will be used in the v1/models API response
    # - if not set, it will be omitted in the JSON model record
    name: "gpt-oss 120B"

    # description: a description for the model
    # - optional, default: empty string
    # - if set, it will be used in the v1/models API response
    # - if not set, it will be omitted in the JSON model record
    description: "A thinking model from OpenAI"

    # env: define an array of environment variables to inject into cmd's environment
    # - optional, default: empty array
    # - each value is a single string
    # - in the format: ENV_NAME=value
    env:
      - "CUDA_VISIBLE_DEVICES=0,1,2"

    # proxy: the URL where llama-swap routes API requests
    # - optional, default: http://localhost:${PORT}
    # - if you used ${PORT} in cmd this can be omitted
    # - if you use a custom port in cmd this *must* be set
    proxy: http://127.0.0.1:8999

    # checkEndpoint: URL path to check if the server is ready
    # - optional, default: /health
    # - endpoint is expected to return an HTTP 200 response
    # - all requests wait until the endpoint is ready or fails
    # - while loading, a non-200 response with a JSON body like
    #   {"progress": 0.42, "message": "loading tensors"} reports loading progress
    #   (progress is 0 to 1) that is shown in the UI
    # - use "none" to skip endpoint health checking
    checkEndpoint: /custom-endpoint

    # ttl: automatically unload the model after ttl seconds
    # - optional, default: -1 (use global default)
    # - ttl values must be a value greater than or equal to 0
    # - a ttl of -1 will use the global TTL value as the default
    # - a ttl of 0 will mean never unload
    # - a value of 0 disables automatic unloading of the model
    ttl: 60

    # unloadTimeout: graceful timeout in seconds when unloading this model (manual, API, or ttl expiry)
    # - optional, default: global unloadTimeout
    # - a value of 0 uses the global unloadTimeout
    # - useful for slow cmdStop commands such as docker stop
    unloadTimeout: 10

    # compat: compatibility settings for upstream applications
    # - optional, default: empty
    compat:
      # ignoreWebsockets: do not let websocket connections load a stopped model
      # or count toward concurrency, TTL activity, or swap decisions
      # - optional, default: false
      ignoreWebsockets: true

    # useModelName: override the model name that is sent to upstream server
    # - optional, default: ""
    # - useful for when the upstream server expects a specific model name that
    #   is different from the model's ID
    useModelName: "openai/gpt-oss-120B"

    # filters: a dictionary of filter settings
    # - optional, default: empty dictionary
    # - same capabilities as peer filters (stripParams, setParams)
    filters:
      # stripParams: a comma separated list of parameters to remove from the request
      # - optional, default: ""
      # - useful for server side enforcement of sampling parameters
      # - the `model` parameter can never be removed
      # - can be any JSON key in the request body
      # - recommended to stick to sampling parameters
      stripParams: "temperature, top_p, top_k"

      # setParams: a dictionary of parameters to set/override in requests
      # - optional, default: empty dictionary
      # - useful for enforcing specific parameter values
      # - protected params like "model" cannot be overridden
      # - values can be strings, numbers, booleans, arrays, or objects
      # - always runs for the model
      # - a key ending in "?" is set-if-undefined: only applied when the
      #   request does not already contain that parameter
      setParams:
        # Example: enforce specific sampling parameters
        temperature: 0.7
        top_p: 0.9
        # Example: default max_tokens, requests that set it win
        max_tokens?: 4096

      # setParamsByID: a dictionary of parameters to set based the model ID
      # - optional, default: empty dictionary
      # - combine with aliases to create variant behaviour without reloading the model
      # - parameters are set in the request body JSON
      # - run after setParams so it will override any settings
      # - protected params like "model" cannot be overridden
      # - values can be strings, numbers, booleans, arrays, or objects
      # - model aliases will be automatically created for each key
      setParamsByID:
        "${MODEL_ID}":
          chat_template_kwargs:
            reasoning_effort: medium
        "${MODEL_ID}:high":
          chat_template_kwargs:
            reasoning_effort: high
        "${MODEL_ID}:low":
          chat_template_kwargs:
            reasoning_effort: low

    # aliases: alternative model names that this model configuration is used for
    # - optional, default: empty array
    # - aliases must be unique globally
    # - useful for impersonating a specific model
    aliases:
      - "gpt-4o-mini"

    # metadata: a dictionary of arbitrary values that are included in /v1/models
    # - optional, default: empty dictionary
    # - while metadata can contains complex types it is recommended to keep it simple
    # - metadata is only passed through in /v1/models responses
    metadata:
      # port will remain an integer
      port: ${PORT}

      # the ${temp} macro will remain a float
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port ${PORT} temp=${temp},
        context=${default_ctx}"

      a_list:
        - 1
        - 1.23
        - "macros are OK in list and dictionary types: ${MODEL_ID}"

      an_obj:
        a: "1"
        b: 2
        # objects can contain complex types with macro substitution
        # becomes: c: [0.7, false, "model: llama"]
        c: ["${temp}", false, "model: ${MODEL_ID}"]

    # concurrencyLimit: overrides the allowed number of active parallel requests to a model
    # - optional, default: 0
    # - useful for limiting the number of active parallel requests a model can process
    # - must be set per model
    # - any number greater than 0 will override the internal default value of 10
    # - any requests that exceeds the limit will receive an HTTP 429 Too Many Requests response
    # - recommended to be omitted and the default used
    concurrencyLimit: 0

    # sendLoadingState: overrides the global sendLoadingState setting for this model
    # - optional, default: undefined (use global setting)
    sendLoadingState: false

    # timeouts: configure proxy connection timeouts for this model
    # - optional, defaults shown below
    # - useful for models running on slower hardware that need longer timeouts
    # - connect: TCP dial connection timeout in seconds, default: 30 seconds
    # - keepalive: TCP connection keepalive timeout, default: 30 seconds
    # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
    # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
    # - idleConn: idle connection timeout in seconds, default: 90 seconds
    # - set any value to 0 to disable that timeout (not recommended)
    timeouts:
      connect: 30
      keepalive: 0
      responseHeader: 60
      tlsHandshake: 10
      idleConn: 90

    # disableKeepAlives: open a new upstream connection for every request
    # - optional, default: false
    # - works around upstreams that advertise keep-alive and then close the
    #   connection, such as llama-server after a streamed response (#1205).
    #   Symptom: a request right after a stream fails with 502 "proxy error: EOF"
    # - when true, idle connections are never pooled, so timeouts.idleConn has
    #   no effect for this model
    disableKeepAlives: false

    # capabilities: defines what the model accepts for input, output and other metadata
    # - optional; omitted or all-zero means no capabilities
    # - used in v1/models to inform clients what the model can do
    # - any field left unset is filled in automatically by asking the upstream
    #   server when the model becomes ready (llama-server and vllm today)
    # - values set here win, field by field; note a field set to its zero
    #   value (false, 0, []) reads as unset, so use disableAuto to suppress
    #   a discovered value rather than writing the zero value
    capabilities:
      # in: list of modalities understood by the model
      # - default: []
      # - valid: text, audio, image, video
      in:
        - text
        - audio
        - image
      # out: list of modalities generated by the model
      # - default: []
      # - valid: text, audio, image, video, decisions
      # - decisions: a llama.cpp decision model, served on /v1/systemone
      out:
        - text
        - audio
        - image
      # tools: the model supports function calling
      # - default: false
      tools: true

      # reranker: the model supports the /v1/rerank endpoint
      # - default: false
      reranker: false

      # context: the maximum token context length supported
      # - default: 0
      # - must be an integer > 0
      context: 32000

      # voices: list of voice names for a speech model
      # - default: []
      # - an override for GET /v1/audio/voices?model={id}
      # - when set, these are returned instead of asking the upstream server,
      #   and the model is not started to answer
      # - when empty or unset, the request goes to the upstream server
      # - the response is {"voices": [...], "model": "{id}"}
      # - not discovered and not shown in v1/models
      voices:
        - voicea
        - voiceb
        - voicec

      # disableAuto: turn off automatic capability discovery for this model
      # - default: false
      # - a field set to its zero value (false, 0, []) cannot be told apart
      #   from an unset one, so use this to suppress a wrong discovered value
      disableAuto: false

  # Unlisted model example:
  "unlisted-model":
    # unlisted: boolean, true or false
    # - optional, default: false
    # - unlisted models do not show up in /v1/models api requests
    # - can be requested as normal through all apis
    unlisted: true
    cmd: some-upstream-server

  # Docker example:
  # container runtimes like Docker and Podman can be used reliably with
  # a combination of cmd, cmdStop, and ${MODEL_ID}
  "docker-llama":
    proxy: "http://127.0.0.1:${PORT}"
    cmd: |
      docker run --name ${MODEL_ID}
      --init --rm -p ${PORT}:8080 -v /mnt/nvme/models:/models
      ghcr.io/ggml-org/llama.cpp:server
      --model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'

    # cmdStop: command to run to stop the model gracefully
    # - optional, default: ""
    # - useful for stopping commands managed by another system
    # - the upstream's process id is available in the ${PID} macro
    #
    # When empty, llama-swap has this default behaviour:
    # - on POSIX systems: a SIGTERM signal is sent
    # - on Windows, calls taskkill to stop the process
    # - processes have 5 seconds to shutdown until forceful termination is attempted
    cmdStop: docker stop ${MODEL_ID}

# hooks: a dictionary of event triggers and actions
# - optional, default: empty dictionary
# - the only supported hook is on_startup
hooks:
  # on_startup: a dictionary of actions to perform on startup
  # - optional, default: empty dictionary
  on_startup:
    # profile: profile to activate on startup and after a configuration reload
    # - optional, default: empty string, which starts with no profile active
    # - must name a key under profiles or the configuration fails to load
    # - runtime selection is not persisted; restarts and reloads return to this profile
    profile: "coding"

    # preload: a list of model ids to load on startup
    # - optional, default: empty list
    # - model names must match keys in the models sections
    # - when preloading multiple models at once, define a group
    #   otherwise models will be loaded and swapped out
    preload:
      - "llama"

# routing:
# Controls how llama-swap decides which models can run at the same time and
# which get swapped out. Choose one of three swap engines:
#
# - group:  the default engine. Simpler to configure. You define groups of
#           models that run together, and loading one group typically unloads
#           the others.
#
# - matrix: the newer engine. More involved to configure, but far more
#           flexible. It uses a small expression language to describe which
#           model combinations are allowed to run concurrently, enabling
#           setups that groups cannot express.
#
# - capacity: declare resource pools and each local model's requirements.
#             Any combination that fits every pool's limit may coexist.
#
# The routing section is optional.
routing:
  router:
    # use: a string defining which engine to use
    # - optional, default: "group"
    # - valid values: group, matrix, capacity
    use: group

    # settings: a dictionary of settings for the specific engines
    settings:
      # capacity: independent resource limits (no automatic measurement)
      # - used when routing.router.use is capacity; other settings may remain
      # - cannot combine with legacy top-level groups/matrix either
      # - pools: dictionary of names to positive integer limits
      # - units are operator-chosen per pool, matching models.*.capacities
      # - unknown pools, requirements exceeding a limit, and invalid values fail
      # - minimizes summed evict_cost, then eviction count, then sorted IDs;
      #   a bounded search may return a feasible nonoptimal plan
      # - only resident models are searched; only the requested model is started
      # capacity:
      #   pools:
      #     ram: 128000
      #     gpu0: 24000
      #     gpu1: 24000

      # groups: a dictionary of named groups
      # - optional, default: empty dictionary
      # - lets you keep some models loaded while others swap out
      # - every member must be a model ID defined in the models section
      # - a model can belong to only one group
      # - behaviour is set per group with the `swap`, `exclusive` and
      #   `persistent` fields
      # - see issue #109 for details
      #
      # NOTE: the model names below are illustrative and are not defined above.
      groups:
        # group1 reproduces llama-swap's default behaviour: only one model
        # runs at a time across the entire instance.
        "group1":
          # swap: how members of this group swap among themselves
          # - optional, default: true
          # - true:  only one member runs at a time
          # - false: all members can run together, no swapping
          swap: true

          # exclusive: how this group affects other groups
          # - optional, default: true
          # - true:  running a member unloads every other group
          # - false: running a member leaves other groups untouched
          exclusive: true

          # members: the model IDs in this group
          # required
          members:
            - "llama"
            - "qwen-unlisted"

        # group2: members all run together, but loading any other group
        # unloads them.
        "group2":
          # swap: false lets all members stay loaded at once
          swap: false

          # exclusive: false means requesting a member loads it without
          # unloading any other group
          exclusive: false
          members:
            - "docker-llama"
            - "modelA"
            - "modelB"

        # forever: a persistent group that other groups can never unload.
        "forever":
          # persistent: other groups cannot unload this group's members
          # - optional, default: false
          # - has no effect on swapping within the group
          persistent: true

          # swap/exclusive: false keeps all members loaded and avoids
          # unloading other groups
          swap: false
          exclusive: false
          members:
            - "forever-modelA"
            - "forever-modelB"
            - "forever-modelc"

      # The matrix lists the model combinations that are allowed to run
      # concurrently. When a model is requested, the solver makes room for it
      # by evicting as few running models as possible, preferring to keep the
      # costliest ones loaded.
      #
      # Solver behaviour:
      #   1. A request arrives for model X.
      #   2. If X is already running, forward the request. Done.
      #   3. Collect every set that contains X.
      #   4. For each set, add up the evict_costs of the running models that
      #      are NOT in that set — that is the set's cost.
      #   5. Choose the lowest-cost set. Break ties by definition order.
      #   6. Evict the models outside that set, start X, forward the request.
      #
      # Subset semantics: a set [a, b, c] also permits any subset of itself.
      # Only the requested model is started; the others are not preloaded.
      #
      # A model that appears in no set can only run on its own.
      #
      matrix:
        # vars: optional aliases for model IDs
        # - names may contain alphanumeric, "-", or "." characters (1-32 chars)
        # - map each short name to a real model ID (not a model alias)
        # - keeps the set expressions short and readable
        # - sets and evict_costs may mix vars with real model IDs
        # - a var takes precedence when its name is also a real model ID
        vars:
          g: gemma-model
          q: qwen-model
          m: mistral-model
          v: voxtral-model
          e: reranker-model

        # evict_costs: relative cost of losing a running model (default: 1)
        evict_costs:
          v: 50 # vllm backend, slow cold start
          llama-70B: 30 # 70B weights, slow to load

        # sets: named combinations of models that may run together.
        # Each value is an expression built from these operators:
        #   &     AND   (models run together)
        #   |     OR    (alternatives)
        #   ()    grouping
        #   +ref  inline the expression of another set
        #
        # Each expression expands into one or more concrete sets:
        #   "model-a"            → [model-a]
        #   "a & b"              → [a, b]
        #   "a | b"              → [a], [b]
        #   "(a | b) & c"        → [a, c], [b, c]
        #   "(a | b) & (c | d)"  → [a,c], [a,d], [b,c], [b,d]
        #   "+llms & v"          → inline the llms set, then AND with v
        sets:
          # +auto:orphans includes each model that is not a leaf in a set.
          # If you define a set with this name, llama-swap uses your set.
          # Example of a catch-all set:
          # always: "task-small & embed-model"
          # main: "+always & moe-medium & (dense-a | dense-b)"
          # scratch: "+always & +auto:orphans"

          # An LLM plus TTS. Vars and real model IDs may be mixed.
          # expands to: [g,v], [qwen-model,v], [m,v]
          standard: "(g | qwen-model | m) & v"

          # An LLM plus TTS plus reranker.
          # expands to: [g,v,e], [q,v,e]
          with_rerank: "(g | q) & v & e"

          # An LLM plus image generation, no TTS.
          # expands to: [g,stable-diffusion], [q,stable-diffusion]
          creative: "(g | q) & stable-diffusion"

          # The 70B model uses every GPU, so it can only run alone.
          # Vars are not required when using real model IDs.
          full: "llama-70B"

  # scheduler: how queued requests are ordered.
  # The default and only valid scheduler is "fifo"
  scheduler:
    use: fifo
    settings:
      fifo:
        # priority: a dictionary of model ID -> priority
        # - optional, default: empty dictionary
        # - models default to priority 0
        # - higher priority requests are serviced first in the queue
        priority:
          A: 10
          B: 5
          C: 5
          D: 1

# peers: a dictionary of remote peers and models they provide
# - optional, default empty dictionary
# - peers can be another llama-swap
# - peers can be any server that provides the /v1/ generative api endpoints supported by llama-swap
# - peer models are always addressable as <peer ID>/<model ID>
# - an unqualified model ID is accepted only when exactly one peer provides it
# - local model IDs and aliases take precedence over unqualified peer model IDs
# - fully qualified peer names are reserved and are the peer IDs shown by /v1/models
peers:
  # keys is the peer'd ID
  llama-swap-peer:
    # proxy: a valid base URL to proxy requests to
    # - required
    # - requested path to llama-swap will be appended to the end of the proxy value
    proxy: http://192.168.1.23
    # models: a list of models served by the peer
    # - required
    models:
      - model_a
      - model_b
      - embeddings/model_c
  # Tailcat peers use the case-sensitive connection token as their proxy.
  # Virtual TCP port 80 is fixed and must not be included in the URL.
  tailcat-peer:
    proxy: tailcat://tcREPLACE_WITH_CONNECTION_TOKEN
    # Optional. Omit (or use "ephemeral") for a process-lifetime client key.
    tailcatKey: /path/to/client.private.json
    # Tailcat peers wait 300s for response headers by default so cold starts
    # can finish; override with timeouts.responseHeader when needed.
    models:
      - model_a
  openrouter:
    proxy: https://openrouter.ai/api
    # apiKey: a string key to be injected into the request
    # - optional, default: ""
    # - if blank, no key will be added to the request
    # - key will be injected into headers: Authorization: Bearer <key> and x-api-key: <key>
    # - can be a string or a macro
    apiKey: ${env.OPENROUTER_API_KEY}
    models:
      - meta-llama/llama-3.1-8b-instruct
      - qwen/qwen3-235b-a22b-2507
      - deepseek/deepseek-v3.2
      - z-ai/glm-4.7
      - moonshotai/kimi-k2-0905
      - minimax/minimax-m2.1
    # timeouts: configure proxy connection timeouts for this peer
    # - optional, defaults shown below
    # - useful when the peer runs on slower hardware
    # - set any value to 0 to disable that timeout (not recommended)
    timeouts:
      connect: 30
      keepalive: 30
      responseHeader: 60
      tlsHandshake: 10
      idleConn: 90

    # filters: a dictionary of filter settings for peer requests
    # - optional, default: empty dictionary
    # - same capabilities as model filters (stripParams, setParams)
    filters:
      # stripParams: a comma separated list of parameters to remove from the request
      # - optional, default: ""
      # - useful for removing parameters that the peer doesn't support
      # - the `model` parameter can never be removed
      stripParams: "temperature, top_p"

      # setParams: a dictionary of parameters to set/override in requests to this peer
      # - optional, default: empty dictionary
      # - useful for injecting provider-specific settings like data retention policies
      # - protected params like "model" cannot be overridden
      # - values can be strings, numbers, booleans, arrays, or objects
      setParams:
        # Example: enforce zero-data-retention for OpenRouter
        provider:
          data_collection: "deny"
          zdr: true