AI - Artificial intelligence

Hey, thanks for the reply, the speed is actually pretty good!

So it's running on llama.cpp (llama-server) b10408, Vulkan/RADV, behind llama-swap.

Your questions made me realize i also had a mismatch in my pi model config, the contextWindow and maxTokens had drifted apart pretty significantly so I just fixed that now.

/etc/nixos/configuration.nix

  services.llama-swap = {
    enable = true;
    package = pkgs.llama-swap;
    listenAddress = "0.0.0.0";
    port = 8080;
    openFirewall = true;

    settings = {
      # 63 GB off NVMe cold is slow. The default 120s will time out.
      healthCheckTimeout = 600;
      logLevel = "info";

      models = {
        "glm-flash" = {
          # Keep "local" as an alias so existing clients keep working
          # without touching their base URL or model name.
          aliases = [ "local" "fast" ];
          ttl = 1800;
          cmd = ''
            ${lib.getExe' pkgs.llama-cpp-vulkan "llama-server"}
            --port ''${PORT}
            -m /srv/llm/models/GLM-4.7-Flash-Q8_0.gguf
            -ngl 999 --flash-attn on -c 131072 -np 1
            --jinja --no-webui
            --reasoning-format auto --reasoning-preserve
            --temp 1.0 --top-p 0.95 --repeat-penalty 1.0
            --cors-origins "http://optimus.local:8080,http://localhost:8080/"
          '';
        };

        "gpt-oss-120b" = {
          aliases = [ "smart" ];
          ttl = 1800;
          cmd = ''
            ${lib.getExe' pkgs.llama-cpp-vulkan "llama-server"}
            --port ''${PORT}
            -m /srv/llm/models/gpt-oss-120b/gpt-oss-120b-MXFP4.gguf
            -ngl 999 --flash-attn on -c 65536 -np 1
            --jinja --no-webui --reasoning-format auto
            --temp 1.0 --top-p 1.0 --top-k 0 --min-p 0.0
            --cors-origins "http://optimus.local:8080,http://localhost:8080/"
          '';
        };
      };
    };
  };

~/.pi/agent/models.json (fixed)

{
  "providers": {
    "optimus": {
      "baseUrl": "http://localhost:8080/v1",
      "api": "openai-completions",
      "apiKey": "ollama",
      "models": [
        {
          "id": "local",
          "name": "GLM-4.7-Flash Q8_0 (optimus)",
          "aliases": ["fast", "glm-flash"],
          "reasoning": true,
          "contextWindow": 131072,
          "maxTokens": 32768,
          "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
        },
        {
          "id": "smart",
          "name": "GPT-OSS-120B (optimus)",
          "aliases": ["gpt-oss-120b", "oss"],
          "reasoning": true,
          "contextWindow": 65536,
          "maxTokens": 16384,
          "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
        }
      ]
    }
  }
}