{
  "collected_at_utc": "2026-10-08T17:26:06.473092+00:00",
  "ollama_version": "0.40.1",
  "hardware": {
    "chip": "Apple M3",
    "memory_gb": 24
  },
  "protocol": {
    "history": false,
    "model_order": [
      "nimble",
      "clef-flash:9b",
      "tev1:4b",
      "laya:421m-english-mlx-fp16"
    ],
    "parameters": "Server defaults; no temperature, seed, or confidence threshold override.",
    "option_order": [
      "cancel_order",
      "check_order_status",
      "insufficient_information"
    ],
    "dataset_unchanged": true,
    "completed_responses_per_model_case": 1,
    "collection": "Original Nimble/Clef Flash records preserved; Tev1/Laya added later using the same inputs and answer key.",
    "unsuccessful_attempts": [
      {
        "model": "tev1:4b",
        "case_id": "intent-01",
        "error": "Connection closed without a response while Ollama restarted. No answer was received. The run was restarted before any successful records."
      }
    ],
    "original_runtime_version": "Not captured at original run time; version 0.40.1 applies to the added runs.",
    "interpretation": "A fixed-policy exploratory test with synthetic messages, not a held-out benchmark. No confidence threshold or calibration evaluation."
  },
  "models": [
    {
      "tag": "nimble:latest",
      "digest": "13ae0a50610239c7cc52589608fc86f8405c5cf1daddd98798771a159e549792",
      "size_bytes": 9267567024,
      "details": {
        "parent_model": "",
        "format": "safetensors",
        "family": "qwen3_5",
        "families": [
          "qwen3_5"
        ],
        "parameter_size": "9.0B",
        "quantization_level": "mxfp8",
        "runner": "mlx"
      },
      "identity_collection": "Inspected during the extension; not captured at original run time.",
      "request_model": "nimble",
      "display_name": "Nimble 9B",
      "source": "https://ollama.com/library/nimble"
    },
    {
      "tag": "clef-flash:9b",
      "digest": "0a2a05d6a581223a9397e6fd2ea92486d4b1a54ec1125065be0c2170ff97ba75",
      "size_bytes": 12387093750,
      "details": {
        "parent_model": "",
        "format": "safetensors",
        "family": "",
        "families": null,
        "parameter_size": "",
        "quantization_level": "mxfp8",
        "runner": "mlx"
      },
      "identity_collection": "Inspected during the extension; not captured at original run time.",
      "request_model": "clef-flash:9b",
      "display_name": "Clef Flash 9B",
      "source": "https://ollama.com/library/clef-flash"
    },
    {
      "tag": "tev1:4b",
      "digest": "4d24c6f6d61a48d9b72902f4805c5f1da53010736b834b0372377fb4bb7e53d3",
      "size_bytes": 4361568327,
      "details": {
        "parent_model": "",
        "format": "safetensors",
        "family": "qwen3_5",
        "families": [
          "qwen3_5"
        ],
        "parameter_size": "4.2B",
        "quantization_level": "mxfp8",
        "runner": "mlx"
      },
      "model_info": {
        "general.architecture": "qwen3_5",
        "general.parameter_count": 4205751296,
        "qwen3_5.block_count": 32,
        "qwen3_5.context_length": 262144,
        "qwen3_5.embedding_length": 2560
      },
      "capabilities": [
        "decision"
      ],
      "identity_collection": "Captured immediately before the added run.",
      "request_model": "tev1:4b",
      "display_name": "Tev1 4B",
      "source": "https://ollama.com/library/tev1"
    },
    {
      "tag": "laya:421m-english-mlx-fp16",
      "digest": "aaba5574a1e14592cb20c64b96292ffa524b3f9ca27263687b87161eacb8d31d",
      "size_bytes": 846206614,
      "details": {
        "parent_model": "",
        "format": "safetensors",
        "family": "",
        "families": null,
        "parameter_size": "",
        "quantization_level": "",
        "runner": "mlx"
      },
      "model_info": {
        "general.architecture": "laya",
        "general.parameter_count": 421275139,
        "laya.attention.head_count": 16,
        "laya.block_count": 28,
        "laya.context_length": 512,
        "laya.embedding_length": 1024,
        "laya.vocab_size": 50368
      },
      "capabilities": [
        "decision"
      ],
      "identity_collection": "Captured immediately before the added run.",
      "request_model": "laya:421m-english-mlx-fp16",
      "display_name": "Laya 421M",
      "source": "https://ollama.com/library/laya"
    }
  ],
  "hosted_reference": {
    "name": "Jev via TryJevAI",
    "endpoint": "https://tryjevai.com/api/jev/evaluate",
    "resolved_model": "jev-1.13.0",
    "provider_reported": "jev",
    "parameter_count": "not verified",
    "source": "Independent third-party playground; underlying request formatting is not independently verified.",
    "mapping": {
      "state": "Unchanged full routing policy, followed by unchanged customer message.",
      "question": "Using the routing policy, classify the customer's currently requested action.",
      "choices": [
        "cancel_order",
        "check_order_status",
        "insufficient_information"
      ]
    },
    "reason_for_mapping": "Original full question was rejected with HTTP 422: maximum 300 characters.",
    "collection": "One successful recorded answer for each of 18 cases. Same expected labels as local runs.",
    "rate_limiting": {
      "http_429_responses": 2,
      "handling": "Waited at least 60 seconds when asked; subsequent calls paced at 11 seconds plus request time."
    },
    "timing": "UI shows response.latencyMs from playground. duration_seconds in raw records is client HTTP elapsed time for successful requests; rate-limit waits excluded. Neither is compared as a speed benchmark.",
    "started_at_utc": "2026-10-08T17:55:48.006517+00:00",
    "finished_at_utc": "2026-10-08T17:59:52.162051+00:00"
  }
}