{ "schema": 1, "parameters": { "werk": "target/debug/werk", "config": "/tmp/werk-mgpu-cpu-layers.json", "out": "/tmp/werk-mgpu-cpu-layer-results.json", "repetitions": 3, "tokens": 16, "prompt": "Return the integer result of 17 + 25, with no explanation.", "required_gpus": 1, "timeout": 600 }, "inventory_before": { "devices": [ { "id": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "physical_index": 0, "vendor": "nvidia", "name": "NVIDIA GeForce RTX 3090", "architecture": "sm_86", "runtime": "cuda", "pci_bus_id": "00000000:03:00.0", "total_bytes": 25769803776, "free_bytes": 21751660544, "numa_node": null, "pcie_link": null, "visible_index": 0 } ], "host_available_bytes": 98456473600, "topology": null, "warnings": [] }, "werk_revision": "8d8ee73a002e8473818d1e86099e0a21b3b51d23", "dirty_diff": " .gitignore | 3 +-\n src/api.rs | 1 +\n src/api/chat.rs | 44 ++++++++++--\n src/api/extended.rs | 22 ++++--\n src/api/generation.rs | 26 ++++++-\n src/api/observability.rs | 3 +-\n src/api/router.rs | 1 +\n src/api/state.rs | 48 +++++++++++++\n src/api/werk/mod.rs | 40 +++++++++++\n src/backend/llama_process_lifecycle.rs | 34 +++++++--\n src/backend/llama_server.rs | 77 ++++++++++++++++++--\n src/backend/llama_server/fp4.rs | 8 +++\n src/backend/llama_server/runtime_state.rs | 116 ++++++++++++++++--------------\n src/backend/vllm.rs | 76 +++++++++++++++++++-\n src/backend/vllm/runtime_control.rs | 6 +-\n src/cli.rs | 59 ++++++++++++++-\n src/inference_service.rs | 3 +-\n src/inference_service/resources.rs | 92 ++++++++++++++++++++++++\n src/lib.rs | 1 +\n 19 files changed, 574 insertions(+), 86 deletions(-)\n", "cold_definition": "fresh worker; OS page cache not flushed", "samples": [ { "alias": "qwen-cpu-layers", "session": "validation-qwen-cpu-layers", "ttft_seconds": 30.996033622999676, "total_seconds": 32.02116944499721, "usage": { "completion_tokens": 3, "prompt_tokens": 29, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 32 }, "text": "42", "repetition": 0, "worker_state": "cold" }, { "alias": "qwen-cpu-layers", "session": "validation-qwen-cpu-layers", "ttft_seconds": 2.061869877998106, "total_seconds": 3.1279374119985732, "usage": { "completion_tokens": 3, "prompt_tokens": 29, "prompt_tokens_details": { "cached_tokens": 25 }, "total_tokens": 32 }, "text": "42", "repetition": 1, "worker_state": "warm" }, { "alias": "qwen-cpu-layers", "session": "validation-qwen-cpu-layers", "ttft_seconds": 1.9183168019990262, "total_seconds": 2.771101770002133, "usage": { "completion_tokens": 3, "prompt_tokens": 29, "prompt_tokens_details": { "cached_tokens": 25 }, "total_tokens": 32 }, "text": "42", "repetition": 2, "worker_state": "warm" } ], "hardware_cases": { "single": "configured", "two_equal": "requires explicit config and >=2 GPUs", "two_unequal": "requires explicit config and >=2 unequal GPUs", "independent": "requires explicit config and >=2 GPUs", "mixed": "requires explicit config and >=3 GPUs", "replicas": "requires explicit config and >=2 GPUs" }, "plans": [ { "profile": { "id": "qwen-cpu-layers", "alias": "multi-gpu-smoke", "model": "qwen3.8-27b-ud-q4_k_xl", "runtime": "llama_cuda", "executable": "/home/philipbodenbach/.local/share/werk1112/backends/llama-cuda/upstream-ec928150501c/build/bin/llama-server", "build": "ec928150501c", "gpus": [ "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9" ], "strategy": "single", "tensor_parallel": 1, "pipeline_parallel": 1, "split": [], "context": 2048, "batch": 256, "parallel": 1, "cpu_threads": 8, "offload": { "kind": "cpu_layers", "gpu_layers": 10 }, "kv_cpu": false, "memory": [ { "weights": 4500000000, "replicated": 0, "kv": 268435456, "experts": 0, "compute": 536870912, "overhead": 268435456, "reserve": 536870912 } ], "host_bytes": 24000000000, "native_args": [ "--no-warmup" ] }, "model_identity": "sha256:1352bb293f26ccf8ee2fd8d226093eff8357e5eac21f078dca9f0e8bc2f6f071", "executable_sha256": "236d00dc1c71b01bd15a13b229edeac2e4b05a9ac740d33aadc14fe6f71287b6", "devices": [ { "id": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "physical_index": 0, "vendor": "nvidia", "name": "NVIDIA GeForce RTX 3090", "architecture": "sm_86", "runtime": "cuda", "pci_bus_id": "00000000:03:00.0", "total_bytes": 25769803776, "free_bytes": 21754806272, "numa_node": null, "pcie_link": null, "visible_index": 0 } ], "strategy": "single", "args": [ "--device", "CUDA0", "--split-mode", "none", "--main-gpu", "0", "--no-warmup" ], "environment": { "CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "OMP_NUM_THREADS": "8" }, "fingerprint": "cdc0ee766e3c4233845e146ecd1da920b609a4c7bc71831d2875cb2d30683734", "reasons": [ "Exclusive GPU ownership; component budgets are declared estimates, not measured usage or hard allocator limits.", "Unknown P2P/NVLink is not treated as a prerequisite; no throughput prediction was made." ] } ], "telemetry": [ { "protocol": { "major": 1, "minor": 0 }, "request_id": "req_3Pm_RQFLWwVDCot2FhFiog", "data": { "host_swap_used_bytes": 0, "host_memory_free_bytes": 73242800128, "schema_version": 1, "observed_at_ms": 1791310863375, "server_started_ms": 1791310829027, "uptime_seconds": 34.348358406, "totals": { "started": 1, "completed": 1, "errors": 0, "cancelled": 0, "active": 0, "prompt_tokens": 29, "output_tokens": 3, "cached_tokens": 0, "duration_seconds": 31.708725212 }, "requests": [ { "id": 1, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310831566, "elapsed_seconds": 31.708725212, "prompt_tokens": 29, "cached_tokens": 0, "output_tokens": 3, "first_output_seconds": 6.861390387, "decode_tokens_per_second": 2.9302140228322275, "prefill_tokens_per_second": 4.228693581572228 } ], "backends": [ { "backend": "deployment", "instance": "qwen-cpu-layers", "model": "multi-gpu-smoke", "available": true, "counters": {}, "gauges": { "gpu_budget_bytes_estimate": 6110612736.0, "gpu_count": 1.0, "host_budget_bytes_estimate": 24000000000.0, "queue_capacity": 0.0, "slots_available": 1.0 } }, { "backend": "llama.cpp / CUDA", "instance": "llama-67758-http://127.0.0.1:39663", "model": "qwen3.8-27b-ud-q4_k_xl", "available": true, "counters": {}, "gauges": { "active_cached_tokens": 0.0, "active_output_tokens": 0.0, "active_prompt_tokens": 0.0, "context_capacity_tokens": 2048.0, "context_used_tokens": 31.0, "gpu_layers_requested": 10.0, "process_resident_bytes": 14857891840.0, "requests_active": 0.0, "slots_total": 1.0 } } ], "memory": { "observed_at_unix_ms": 1791310863374, "overall_pressure": "normal", "topology": "discrete", "host": { "capacity_bytes": 105469059072, "available_bytes": 98959720448, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "accelerator": { "capacity_bytes": 25769803776, "available_bytes": 17042505728, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "last_action_unix_ms": null, "counters": { "active_reservations": 0, "backend_cleanup_failures": 0, "backend_cleanup_latched": 0, "completed_demotions": 0, "completed_evictions": 0, "failed_releases": 0, "managed_allocations": 0, "orphaned_release_bytes": 0, "pressure_actions_in_flight": 0 } }, "backend_observed_at_ms": 1791310863290 } }, { "protocol": { "major": 1, "minor": 0 }, "request_id": "req_G6U1FIp1lwl60oUQMlWcOA", "data": { "host_swap_used_bytes": 0, "host_memory_free_bytes": 72917770240, "schema_version": 1, "observed_at_ms": 1791310866612, "server_started_ms": 1791310829027, "uptime_seconds": 37.5853276, "totals": { "started": 2, "completed": 2, "errors": 0, "cancelled": 0, "active": 0, "prompt_tokens": 58, "output_tokens": 6, "cached_tokens": 25, "duration_seconds": 34.557393367 }, "requests": [ { "id": 2, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310863656, "elapsed_seconds": 2.848668155, "prompt_tokens": 29, "cached_tokens": 25, "output_tokens": 3, "first_output_seconds": 1.781701921, "decode_tokens_per_second": 2.818497233644965, "prefill_tokens_per_second": 2.625018129031454 }, { "id": 1, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310831566, "elapsed_seconds": 31.708725212, "prompt_tokens": 29, "cached_tokens": 0, "output_tokens": 3, "first_output_seconds": 6.861390387, "decode_tokens_per_second": 2.9302140228322275, "prefill_tokens_per_second": 4.228693581572228 } ], "backends": [ { "backend": "deployment", "instance": "qwen-cpu-layers", "model": "multi-gpu-smoke", "available": true, "counters": {}, "gauges": { "gpu_budget_bytes_estimate": 6110612736.0, "gpu_count": 1.0, "host_budget_bytes_estimate": 24000000000.0, "queue_capacity": 0.0, "slots_available": 1.0 } }, { "backend": "llama.cpp / CUDA", "instance": "llama-67758-http://127.0.0.1:39663", "model": "qwen3.8-27b-ud-q4_k_xl", "available": true, "counters": {}, "gauges": { "active_cached_tokens": 0.0, "active_output_tokens": 0.0, "active_prompt_tokens": 0.0, "context_capacity_tokens": 2048.0, "context_used_tokens": 31.0, "gpu_layers_requested": 10.0, "process_resident_bytes": 15176871936.0, "requests_active": 0.0, "slots_total": 1.0 } } ], "memory": { "observed_at_unix_ms": 1791310866611, "overall_pressure": "normal", "topology": "discrete", "host": { "capacity_bytes": 105469059072, "available_bytes": 98639577088, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "accelerator": { "capacity_bytes": 25769803776, "available_bytes": 17035165696, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "last_action_unix_ms": null, "counters": { "active_reservations": 0, "backend_cleanup_failures": 0, "backend_cleanup_latched": 0, "completed_demotions": 0, "completed_evictions": 0, "failed_releases": 0, "managed_allocations": 0, "orphaned_release_bytes": 0, "pressure_actions_in_flight": 0 } }, "backend_observed_at_ms": 1791310866516 } }, { "protocol": { "major": 1, "minor": 0 }, "request_id": "req_PdiFO-hScjd9vbI3OCrjiw", "data": { "host_swap_used_bytes": 0, "host_memory_free_bytes": 72955060224, "schema_version": 1, "observed_at_ms": 1791310869477, "server_started_ms": 1791310829027, "uptime_seconds": 40.450029756, "totals": { "started": 3, "completed": 3, "errors": 0, "cancelled": 0, "active": 0, "prompt_tokens": 87, "output_tokens": 9, "cached_tokens": 50, "duration_seconds": 37.058780379000005 }, "requests": [ { "id": 3, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310866884, "elapsed_seconds": 2.501387012, "prompt_tokens": 29, "cached_tokens": 25, "output_tokens": 3, "first_output_seconds": 1.646914244, "decode_tokens_per_second": 3.518958388317058, "prefill_tokens_per_second": 2.431643462708924 }, { "id": 2, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310863656, "elapsed_seconds": 2.848668155, "prompt_tokens": 29, "cached_tokens": 25, "output_tokens": 3, "first_output_seconds": 1.781701921, "decode_tokens_per_second": 2.818497233644965, "prefill_tokens_per_second": 2.625018129031454 }, { "id": 1, "model": "qwen3.8-27b-ud-q4_k_xl", "state": "done", "started_ms": 1791310831566, "elapsed_seconds": 31.708725212, "prompt_tokens": 29, "cached_tokens": 0, "output_tokens": 3, "first_output_seconds": 6.861390387, "decode_tokens_per_second": 2.9302140228322275, "prefill_tokens_per_second": 4.228693581572228 } ], "backends": [ { "backend": "deployment", "instance": "qwen-cpu-layers", "model": "multi-gpu-smoke", "available": true, "counters": {}, "gauges": { "gpu_budget_bytes_estimate": 6110612736.0, "gpu_count": 1.0, "host_budget_bytes_estimate": 24000000000.0, "queue_capacity": 0.0, "slots_available": 1.0 } }, { "backend": "llama.cpp / CUDA", "instance": "llama-67758-http://127.0.0.1:39663", "model": "qwen3.8-27b-ud-q4_k_xl", "available": true, "counters": {}, "gauges": { "active_cached_tokens": 0.0, "active_output_tokens": 0.0, "active_prompt_tokens": 0.0, "context_capacity_tokens": 2048.0, "context_used_tokens": 31.0, "gpu_layers_requested": 10.0, "process_resident_bytes": 15176941568.0, "requests_active": 0.0, "slots_total": 1.0 } } ], "memory": { "observed_at_unix_ms": 1791310869475, "overall_pressure": "normal", "topology": "discrete", "host": { "capacity_bytes": 105469059072, "available_bytes": 98671759360, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "accelerator": { "capacity_bytes": 25769803776, "available_bytes": 17035165696, "managed_bytes": 0, "reserved_bytes": 0, "pressure": "normal" }, "last_action_unix_ms": null, "counters": { "active_reservations": 0, "backend_cleanup_failures": 0, "backend_cleanup_latched": 0, "completed_demotions": 0, "completed_evictions": 0, "failed_releases": 0, "managed_allocations": 0, "orphaned_release_bytes": 0, "pressure_actions_in_flight": 0 } }, "backend_observed_at_ms": 1791310869392 } } ], "diagnostics_after": { "deployments": { "profiles": [ { "available_slots": 1, "plan": { "args": [ "--device", "CUDA0", "--split-mode", "none", "--main-gpu", "0", "--no-warmup" ], "devices": [ { "architecture": "sm_86", "free_bytes": 21757952000, "id": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "name": "NVIDIA GeForce RTX 3090", "numa_node": null, "pci_bus_id": "00000000:03:00.0", "pcie_link": null, "physical_index": 0, "runtime": "cuda", "total_bytes": 25769803776, "vendor": "nvidia", "visible_index": 0 } ], "environment": { "CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "OMP_NUM_THREADS": "8" }, "executable_sha256": "236d00dc1c71b01bd15a13b229edeac2e4b05a9ac740d33aadc14fe6f71287b6", "fingerprint": "cdc0ee766e3c4233845e146ecd1da920b609a4c7bc71831d2875cb2d30683734", "model_identity": "sha256:1352bb293f26ccf8ee2fd8d226093eff8357e5eac21f078dca9f0e8bc2f6f071", "profile": { "alias": "multi-gpu-smoke", "batch": 256, "build": "ec928150501c", "context": 2048, "cpu_threads": 8, "executable": "/home/philipbodenbach/.local/share/werk1112/backends/llama-cuda/upstream-ec928150501c/build/bin/llama-server", "gpus": [ "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9" ], "host_bytes": 24000000000, "id": "qwen-cpu-layers", "kv_cpu": false, "memory": [ { "compute": 536870912, "experts": 0, "kv": 268435456, "overhead": 268435456, "replicated": 0, "reserve": 536870912, "weights": 4500000000 } ], "model": "qwen3.8-27b-ud-q4_k_xl", "native_args": [ "--no-warmup" ], "offload": { "gpu_layers": 10, "kind": "cpu_layers" }, "parallel": 1, "pipeline_parallel": 1, "runtime": "llama_cuda", "split": [], "strategy": "single", "tensor_parallel": 1 }, "reasons": [ "Exclusive GPU ownership; component budgets are declared estimates, not measured usage or hard allocator limits.", "Unknown P2P/NVLink is not treated as a prerequisite; no throughput prediction was made." ], "strategy": "single" }, "queue_capacity": 0, "queue_policy": "reject", "workers": [ { "available": true, "backend": "llama.cpp / CUDA", "counters": {}, "gauges": { "active_cached_tokens": 0.0, "active_output_tokens": 0.0, "active_prompt_tokens": 0.0, "context_capacity_tokens": 2048.0, "context_used_tokens": 31.0, "gpu_layers_requested": 10.0, "process_resident_bytes": 15176966144.0, "requests_active": 0.0, "slots_total": 1.0 }, "instance": "llama-67758-http://127.0.0.1:39663", "model": "qwen3.8-27b-ud-q4_k_xl" } ] } ] }, "inventory": { "devices": [ { "architecture": "sm_86", "free_bytes": 17035165696, "id": "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9", "name": "NVIDIA GeForce RTX 3090", "numa_node": null, "pci_bus_id": "00000000:03:00.0", "pcie_link": null, "physical_index": 0, "runtime": "cuda", "total_bytes": 25769803776, "vendor": "nvidia", "visible_index": 0 } ], "host_available_bytes": 98644819968, "topology": null, "warnings": [] } }, "status": "passed_transport", "correctness": "Outputs retained for reference comparison; transport success alone is not numerical equivalence.", "latency_seconds": { "median": 3.1279374119985732, "min": 2.771101770002133, "max": 32.02116944499721 }, "gpu_memory_samples": [ { "time": 1791310828.1942096, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3555 } }, { "time": 1791310828.545053, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3555 } }, { "time": 1791310828.8483853, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3575 } }, { "time": 1791310829.1749725, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3576 } }, { "time": 1791310829.4480627, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3576 } }, { "time": 1791310829.7478514, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3575 } }, { "time": 1791310830.0914135, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3575 } }, { "time": 1791310830.447871, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3575 } }, { "time": 1791310830.7627687, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3570 } }, { "time": 1791310831.1252372, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3567 } }, { "time": 1791310831.5125976, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3567 } }, { "time": 1791310831.8348434, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3583 } }, { "time": 1791310832.1781335, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3579 } }, { "time": 1791310832.5247905, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3568 } }, { "time": 1791310832.845209, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3587 } }, { "time": 1791310833.1454875, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3585 } }, { "time": 1791310833.4715753, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3585 } }, { "time": 1791310833.7719798, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3585 } }, { "time": 1791310834.0673082, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3836 } }, { "time": 1791310834.4030538, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3836 } }, { "time": 1791310834.7012968, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3836 } }, { "time": 1791310835.0281494, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310835.3532026, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310835.6723487, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310835.980224, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3822 } }, { "time": 1791310836.2803931, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3822 } }, { "time": 1791310836.5851324, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3824 } }, { "time": 1791310836.8921492, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3843 } }, { "time": 1791310837.1990063, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3844 } }, { "time": 1791310837.5155287, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3844 } }, { "time": 1791310837.817669, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3844 } }, { "time": 1791310838.1119401, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3841 } }, { "time": 1791310838.4176092, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3841 } }, { "time": 1791310838.7158334, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3841 } }, { "time": 1791310839.022839, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310839.3378644, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310839.6411197, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3833 } }, { "time": 1791310839.9561903, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3816 } }, { "time": 1791310840.2682922, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3816 } }, { "time": 1791310840.567417, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3816 } }, { "time": 1791310840.8602684, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3835 } }, { "time": 1791310841.1742413, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3836 } }, { "time": 1791310841.4631834, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310841.786119, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310842.0779505, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310842.3830073, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310842.7031167, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310843.0036368, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310843.2892969, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310843.5937214, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310843.9172156, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310844.2134502, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3806 } }, { "time": 1791310844.521424, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3806 } }, { "time": 1791310844.8299348, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3825 } }, { "time": 1791310845.1300354, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310845.4177146, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310845.722525, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310846.0234087, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310846.3198948, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310846.627668, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310846.9270496, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3825 } }, { "time": 1791310847.232773, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310847.5261889, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310847.8355439, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310848.1385884, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3806 } }, { "time": 1791310848.4473016, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3806 } }, { "time": 1791310848.7595391, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3827 } }, { "time": 1791310849.0957808, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310849.4081678, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310849.7250416, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310850.0296826, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310850.3601549, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310850.6688533, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3826 } }, { "time": 1791310850.955416, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3825 } }, { "time": 1791310851.2774053, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310851.5752926, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3823 } }, { "time": 1791310851.9114373, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 6383 } }, { "time": 1791310852.2337697, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 6949 } }, { "time": 1791310852.5464926, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 6949 } }, { "time": 1791310852.8661509, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8031 } }, { "time": 1791310853.1862135, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8032 } }, { "time": 1791310853.481671, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8032 } }, { "time": 1791310853.7924843, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8032 } }, { "time": 1791310854.0979776, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8032 } }, { "time": 1791310854.3922145, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8034 } }, { "time": 1791310854.682891, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8034 } }, { "time": 1791310854.980585, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8063 } }, { "time": 1791310855.299445, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310855.5912023, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310855.8740103, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310856.171543, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8048 } }, { "time": 1791310856.4518769, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8048 } }, { "time": 1791310856.73256, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8068 } }, { "time": 1791310857.0271401, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8063 } }, { "time": 1791310857.3091774, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310857.5912952, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310857.8711512, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310858.1535969, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310858.4373713, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310858.7183018, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8064 } }, { "time": 1791310859.0070522, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310859.286022, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310859.5678961, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310859.8483248, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8061 } }, { "time": 1791310860.1353822, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8044 } }, { "time": 1791310860.4225905, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8044 } }, { "time": 1791310860.7028103, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8065 } }, { "time": 1791310861.004958, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8071 } }, { "time": 1791310861.287964, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310861.5727332, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310861.8580687, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310862.1382585, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310862.4476235, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310862.7376854, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8072 } }, { "time": 1791310863.0318685, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8071 } }, { "time": 1791310863.3269706, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8073 } }, { "time": 1791310863.6433752, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8073 } }, { "time": 1791310863.9542744, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8073 } }, { "time": 1791310864.2533228, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8056 } }, { "time": 1791310864.5356154, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8056 } }, { "time": 1791310864.8194497, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8075 } }, { "time": 1791310865.1065388, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8076 } }, { "time": 1791310865.4225137, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310865.720677, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310866.0131328, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310866.3115935, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310866.6142035, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310866.9220834, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310867.2276537, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8077 } }, { "time": 1791310867.511283, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8077 } }, { "time": 1791310867.7979815, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8077 } }, { "time": 1791310868.0845222, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8060 } }, { "time": 1791310868.3694015, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8060 } }, { "time": 1791310868.6746812, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8081 } }, { "time": 1791310868.9675503, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8079 } }, { "time": 1791310869.2639365, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310869.56383, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310869.8662393, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8080 } }, { "time": 1791310870.2309952, "used_mib": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 3570 } } ], "gpu_peak_used_mib_including_other_processes": { "GPU-b41bf955-b7e6-6f64-82b2-6b3cf08450e9": 8081 }, "unavailable_measurements": [ "PCIe transfers", "pinned RAM", "per-worker GPU allocation", "logit/logprob equivalence" ], "installed_llama_revision": "ec928150501c2572fec05cb949061672bb424914" }