{ "hardware": { "build": "25F84", "chip": "Apple M4 Max", "cpuCores": "proc 16:12:4:0", "gpus": [ { "spdisplays_vendor": "sppci_vendor_Apple", "sppci_model": "Apple M4 Max" } ], "macOS": "26.5.2", "memory": "128 GB" }, "measurementMode": "desktop-load", "modelManifests": [ { "components": { "text_encoder": { "path": ".", "type": "local" }, "tokenizer": { "path": ".", "type": "local" } }, "createdAt": "2026-09-01T00:00:00Z", "engine": "qwen3.5-hybrid-moe", "family": "code", "id": "text-agent-ornith-35b-mlx-4bit", "precision": "int4", "quantization": { "bits": 4, "groupSize": 64, "scheme": "mlx-affine" }, "schemaVersion": 3, "sources": [ { "repository": "ornith-ai/Ornith-1.5-35B-A3B-MLX-4bit", "revision": "19504d912fa8fc7622bf6b1de3db5d5d890b1f02", "role": "primary" }, { "destination_path": "vision", "repository": "ornith-ai/Ornith-1.5-35B-A3B", "revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1", "role": "component" }, { "destination_path": "mtp", "repository": "ornith-ai/Ornith-1.5-35B-A3B", "revision": "e4dfb35a93d4b6822a811a7676f3488514abe7e2", "role": "component" } ], "supports": [ "chat", "code_generation", "vision_chat" ], "tier": "large", "upstreamRepoId": "Sawfwair/Ornith-1.5-35B-A3B-MLX-4bit-Vision-MTP@main", "variant": "standard" }, { "components": { "text_encoder": { "path": ".", "type": "local" }, "tokenizer": { "path": ".", "type": "local" } }, "createdAt": "2026-08-29T13:08:53Z", "engine": "qwen3.5-hybrid-moe", "family": "qwen", "id": "vision-chat-q38-27b-4bit", "precision": "int4", "quantization": { "bits": 4, "groupSize": 64, "scheme": "mlx-affine" }, "schemaVersion": 3, "sources": [ { "repository": "EigenLabs/Qwen3.8-27B-4bit", "revision": "eda45ab47f465d08d6558f0353a2346e2eb9d5b3", "role": "primary" }, { "destination_path": "mtp", "repository": "morgan/qwen38-27b-mtp-r20k-lr3-q4-g64-q2-rerank", "revision": "fd4a99c590dd6e468c0e2a28168c235e32151a4b", "role": "component" }, { "destination_path": "vision", "repository": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "role": "component" }, { "destination_path": "licenses/qwen3.8", "repository": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "role": "component" } ], "supports": [ "chat", "code_generation", "vision_chat" ], "tier": "latest", "upstreamRepoId": "EigenLabs/Qwen3.8-27B-4bit@eda45ab47f465d08d6558f0353a2346e2eb9d5b3", "variant": "standard" } ], "notes": [ "All measured repetitions are retained. Model load is warmed separately; complete-request timing includes prefill and request overhead.", "Scheduling selection used the earlier f847be2e binary with explicit environment overrides. Final workloads use the recorded release build with no scheduling overrides.", "Machine-local command paths and process identifiers are omitted. Counts and utilization samples are preserved.", "Fixed-token measurements ignore EOS and do not evaluate completion quality. Sampled runs do not claim greedy parity." ], "records": [ { "id": "finish-selection/ornith-c18/ornith-code", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "ab7286d68ba567842ed1b603798bcbcc41a8956195613f90c48e22ce15d6635a", "receipt": { "after": { "gpuDeviceUtilizationPercent": 90, "loadAverage": [ 5.40087890625, 6.3662109375, 7.38916015625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 5.447265625, 6.4443359375, 7.44091796875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.0815439224243164, "loadAverage": [ 5.447265625, 6.4443359375, 7.44091796875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 2.192143201828003, "loadAverage": [ 5.447265625, 6.4443359375, 7.44091796875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 4.293489217758179, "loadAverage": [ 5.4912109375, 6.4365234375, 7.43212890625 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 6.397712230682373, "loadAverage": [ 5.4912109375, 6.4365234375, 7.43212890625 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 8.50563931465149, "loadAverage": [ 5.37158203125, 6.39599609375, 7.41162109375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 10.607057332992554, "loadAverage": [ 5.37158203125, 6.39599609375, 7.41162109375 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 12.70720911026001, "loadAverage": [ 5.37158203125, 6.39599609375, 7.41162109375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 14.817098140716553, "loadAverage": [ 5.26171875, 6.35595703125, 7.3916015625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 16.92239809036255, "loadAverage": [ 5.26171875, 6.35595703125, 7.3916015625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 19.030346155166626, "loadAverage": [ 5.40087890625, 6.3662109375, 7.38916015625 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9462693888, "processSeconds": 21.13405704498291, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621813.2354639, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9908294243221866 }, "endToEndSpeedups": { "adaptive": 0.9913776188590493 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.8451869487762451, "decodeTokensPerSecond": 138.73932945915476, "elapsedSeconds": 1.9189720153808594, "endToEndTokensPerSecond": 133.4047593962393, "expectedMTPActive": false, "firstTokenSeconds": 0.0074269771575927734, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07226800918579102, "prefillTokensPerSecond": 747.2185910251588, "promptTokens": 54, "residentMemoryAfterBytes": 7693549568, "residentMemoryBeforeBytes": 7693418496, "ttftSeconds": 0.07970094680786133 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 1.8622649908065796, "decodeTokensPerSecond": 137.46700993886049, "elapsedSeconds": 1.935662031173706, "endToEndTokensPerSecond": 132.2544927147081, "expectedMTPActive": true, "firstTokenSeconds": 0.00017404556274414062, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07181298732757568, "prefillTokensPerSecond": 751.9531217059449, "promptTokens": 54, "residentMemoryAfterBytes": 9462546432, "residentMemoryBeforeBytes": 9462497280, "ttftSeconds": 0.07199299335479736 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.955751554872881 }, "endToEndSpeedups": { "adaptive": 0.9573893593369335 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.8309409618377686, "decodeTokensPerSecond": 139.81881739269483, "elapsedSeconds": 1.9049999713897705, "endToEndTokensPerSecond": 134.38320411797076, "expectedMTPActive": false, "firstTokenSeconds": 0.00734400749206543, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07248198986053467, "prefillTokensPerSecond": 745.0126590605947, "promptTokens": 54, "residentMemoryAfterBytes": 7693844480, "residentMemoryBeforeBytes": 7693647872, "ttftSeconds": 0.07983195781707764 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 1.915708065032959, "decodeTokensPerSecond": 133.6320521235555, "elapsedSeconds": 1.9897860288619995, "endToEndTokensPerSecond": 128.6570496961484, "expectedMTPActive": true, "firstTokenSeconds": 0.0001710653305053711, "generatedTokens": 256, "loadSeconds": 6.9141387939453125e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07250797748565674, "prefillTokensPerSecond": 744.7456386530997, "promptTokens": 54, "residentMemoryAfterBytes": 9462595584, "residentMemoryBeforeBytes": 9462546432, "ttftSeconds": 0.07268595695495605 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9224825188672527 }, "endToEndSpeedups": { "adaptive": 0.9247339504748578 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.774337887763977, "decodeTokensPerSecond": 144.27917127025424, "elapsedSeconds": 1.8480299711227417, "endToEndTokensPerSecond": 138.52589189582852, "expectedMTPActive": false, "firstTokenSeconds": 0.007406949996948242, "generatedTokens": 256, "loadSeconds": 7.033348083496094e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07218503952026367, "prefillTokensPerSecond": 748.0774459483562, "promptTokens": 54, "residentMemoryAfterBytes": 7694090240, "residentMemoryBeforeBytes": 7693844480, "ttftSeconds": 0.07959902286529541 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 1.9234379529953003, "decodeTokensPerSecond": 133.0950133334639, "elapsedSeconds": 1.9984450340270996, "endToEndTokensPerSecond": 128.0995952558826, "expectedMTPActive": true, "firstTokenSeconds": 0.0001589059829711914, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07333493232727051, "prefillTokensPerSecond": 736.3475807002201, "promptTokens": 54, "residentMemoryAfterBytes": 9462693888, "residentMemoryBeforeBytes": 9462595584, "ttftSeconds": 0.07349979877471924 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/ornith-c18/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "66e0883b3a0752924f6946858ab5a8f95b9ff41f5245f9136017150a4c865351", "receipt": { "after": { "gpuDeviceUtilizationPercent": 88, "loadAverage": [ 5.6318359375, 6.34912109375, 7.353515625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 5.40087890625, 6.3662109375, 7.38916015625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08645415306091309, "loadAverage": [ 5.40087890625, 6.3662109375, 7.38916015625 ] }, { "deviceUtilizationPercent": 31, "elapsedSeconds": 2.190883159637451, "loadAverage": [ 5.60888671875, 6.39306640625, 7.392578125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 4.295640230178833, "loadAverage": [ 5.60888671875, 6.39306640625, 7.392578125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 6.402127265930176, "loadAverage": [ 5.60888671875, 6.39306640625, 7.392578125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 8.50639009475708, "loadAverage": [ 5.47998046875, 6.35302734375, 7.37255859375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 10.613569021224976, "loadAverage": [ 5.47998046875, 6.35302734375, 7.37255859375 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 12.712785005569458, "loadAverage": [ 5.841796875, 6.4130859375, 7.3876953125 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 14.813679218292236, "loadAverage": [ 5.841796875, 6.4130859375, 7.3876953125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 16.918843984603882, "loadAverage": [ 5.77392578125, 6.38916015625, 7.37353515625 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 19.038222074508667, "loadAverage": [ 5.77392578125, 6.38916015625, 7.37353515625 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 21.148324966430664, "loadAverage": [ 5.77392578125, 6.38916015625, 7.37353515625 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9463054336, "processSeconds": 23.25626015663147, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621834.491798, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.8719640672950005 }, "endToEndSpeedups": { "adaptive": 0.8759395567474784 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 2.1430710554122925, "decodeTokensPerSecond": 119.45474199442711, "elapsedSeconds": 2.221827983856201, "endToEndTokensPerSecond": 115.22044094326637, "expectedMTPActive": true, "firstTokenSeconds": 0.0003650188446044922, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.07698297500610352, "prefillTokensPerSecond": 779.3931060113351, "promptTokens": 60, "residentMemoryAfterBytes": 9462743040, "residentMemoryBeforeBytes": 9462415360, "ttftSeconds": 0.0773540735244751 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.8686809539794922, "decodeTokensPerSecond": 136.99502820683722, "elapsedSeconds": 1.9461870193481445, "endToEndTokensPerSecond": 131.53925982187704, "expectedMTPActive": false, "firstTokenSeconds": 0.007520914077758789, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.07584500312805176, "prefillTokensPerSecond": 791.0870528767717, "promptTokens": 60, "residentMemoryAfterBytes": 9332129792, "residentMemoryBeforeBytes": 9332064256, "ttftSeconds": 0.08337187767028809 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9493325516802852 }, "endToEndSpeedups": { "adaptive": 0.9499885322222813 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 2.113981008529663, "decodeTokensPerSecond": 121.09853350955865, "elapsedSeconds": 2.1933770179748535, "endToEndTokensPerSecond": 116.71500061415112, "expectedMTPActive": true, "firstTokenSeconds": 0.00016307830810546875, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.0777510404586792, "prefillTokensPerSecond": 771.6938531759843, "promptTokens": 60, "residentMemoryAfterBytes": 9462956032, "residentMemoryBeforeBytes": 9462841344, "ttftSeconds": 0.07792007923126221 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.006870985031128, "decodeTokensPerSecond": 127.5617625195918, "elapsedSeconds": 2.0836830139160156, "endToEndTokensPerSecond": 122.85937846125681, "expectedMTPActive": false, "firstTokenSeconds": 0.008145928382873535, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.07517600059509277, "prefillTokensPerSecond": 798.127055510274, "promptTokens": 60, "residentMemoryAfterBytes": 9332162560, "residentMemoryBeforeBytes": 9332129792, "ttftSeconds": 0.08332788944244385 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9290656070891764 }, "endToEndSpeedups": { "adaptive": 0.9322534747827848 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 2.150523066520691, "decodeTokensPerSecond": 119.04080639050282, "elapsedSeconds": 2.2277320623397827, "endToEndTokensPerSecond": 114.91507633603105, "expectedMTPActive": true, "firstTokenSeconds": 0.00031304359436035156, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.07560300827026367, "prefillTokensPerSecond": 793.6192140068496, "promptTokens": 60, "residentMemoryAfterBytes": 9463054336, "residentMemoryBeforeBytes": 9462956032, "ttftSeconds": 0.07592201232910156 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.9979770183563232, "decodeTokensPerSecond": 128.1296019163442, "elapsedSeconds": 2.0768109560012817, "endToEndTokensPerSecond": 123.26591366452807, "expectedMTPActive": false, "firstTokenSeconds": 0.00781095027923584, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.07689595222473145, "prefillTokensPerSecond": 780.2751414619022, "promptTokens": 60, "residentMemoryAfterBytes": 9332178944, "residentMemoryBeforeBytes": 9332162560, "ttftSeconds": 0.08471298217773438 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/ornith-c35/ornith-code", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "49aea00f60c4eb706de442673fe11ea4e08ecf5761a21128f14ef66c5f10e656", "receipt": { "after": { "gpuDeviceUtilizationPercent": 85, "loadAverage": [ 6.79931640625, 6.568359375, 7.408203125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 5.6318359375, 6.34912109375, 7.353515625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08911299705505371, "loadAverage": [ 5.6318359375, 6.34912109375, 7.353515625 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 2.194761037826538, "loadAverage": [ 5.6318359375, 6.34912109375, 7.353515625 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 4.3024938106536865, "loadAverage": [ 6.1416015625, 6.44287109375, 7.38037109375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 6.414725065231323, "loadAverage": [ 6.1416015625, 6.44287109375, 7.38037109375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 8.519077062606812, "loadAverage": [ 6.2900390625, 6.46826171875, 7.3837890625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 10.614816904067993, "loadAverage": [ 6.2900390625, 6.46826171875, 7.3837890625 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 12.72452712059021, "loadAverage": [ 6.2900390625, 6.46826171875, 7.3837890625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 14.835228204727173, "loadAverage": [ 6.3466796875, 6.47705078125, 7.38134765625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 16.95377516746521, "loadAverage": [ 6.3466796875, 6.47705078125, 7.38134765625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 19.067198038101196, "loadAverage": [ 6.79931640625, 6.568359375, 7.408203125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9467527168, "processSeconds": 21.17357587814331, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.35", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621857.881758, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0940898949345736 }, "endToEndSpeedups": { "adaptive": 1.090033853747573 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.9861888885498047, "decodeTokensPerSecond": 128.89005747429982, "elapsedSeconds": 2.060573935508728, "endToEndTokensPerSecond": 124.23723099108163, "expectedMTPActive": false, "firstTokenSeconds": 0.00848388671875, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07282507419586182, "prefillTokensPerSecond": 741.5028490704714, "promptTokens": 54, "residentMemoryAfterBytes": 7684620288, "residentMemoryBeforeBytes": 7684341760, "ttftSeconds": 0.08131492137908936 }, { "acceleration": { "acceptanceRate": 0.6097560975609756, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 205, "rounds": 130, "route": "mtp-speculative" }, "decodeSeconds": 1.8153799772262573, "decodeTokensPerSecond": 141.01730944016785, "elapsedSeconds": 1.890376091003418, "endToEndTokensPerSecond": 135.4227876761361, "expectedMTPActive": true, "firstTokenSeconds": 0.00020396709442138672, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07333004474639893, "prefillTokensPerSecond": 736.3966596059089, "promptTokens": 54, "residentMemoryAfterBytes": 9467412480, "residentMemoryBeforeBytes": 9467346944, "ttftSeconds": 0.07353997230529785 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9023155041251927 }, "endToEndSpeedups": { "adaptive": 0.9033363554453183 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.8812980651855469, "decodeTokensPerSecond": 136.07625752528028, "elapsedSeconds": 1.954930067062378, "endToEndTokensPerSecond": 130.9509758498341, "expectedMTPActive": false, "firstTokenSeconds": 0.007853031158447266, "generatedTokens": 256, "loadSeconds": 7.033348083496094e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.0721750259399414, "prefillTokensPerSecond": 748.1812343917231, "promptTokens": 54, "residentMemoryAfterBytes": 7685128192, "residentMemoryBeforeBytes": 7684718592, "ttftSeconds": 0.08003509044647217 }, { "acceleration": { "acceptanceRate": 0.6097560975609756, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 205, "rounds": 130, "route": "mtp-speculative" }, "decodeSeconds": 2.084967017173767, "decodeTokensPerSecond": 122.78371690839282, "elapsedSeconds": 2.164121985435486, "endToEndTokensPerSecond": 118.29277726619702, "expectedMTPActive": true, "firstTokenSeconds": 0.00020003318786621094, "generatedTokens": 256, "loadSeconds": 9.059906005859375e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07732093334197998, "prefillTokensPerSecond": 698.3878448694526, "promptTokens": 54, "residentMemoryAfterBytes": 9467461632, "residentMemoryBeforeBytes": 9467412480, "ttftSeconds": 0.07753002643585205 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9583593961964133 }, "endToEndSpeedups": { "adaptive": 0.9573203954950966 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.8140190839767456, "decodeTokensPerSecond": 141.1231018798266, "elapsedSeconds": 1.8873590230941772, "endToEndTokensPerSecond": 135.63926993620328, "expectedMTPActive": false, "firstTokenSeconds": 0.007634997367858887, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07190501689910889, "prefillTokensPerSecond": 750.9907142608462, "promptTokens": 54, "residentMemoryAfterBytes": 7685308416, "residentMemoryBeforeBytes": 7685128192, "ttftSeconds": 0.07954597473144531 }, { "acceleration": { "acceptanceRate": 0.6097560975609756, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 205, "rounds": 130, "route": "mtp-speculative" }, "decodeSeconds": 1.8928380012512207, "decodeTokensPerSecond": 135.24665070691555, "elapsedSeconds": 1.9715019464492798, "endToEndTokensPerSecond": 129.8502395399923, "expectedMTPActive": true, "firstTokenSeconds": 0.0002410411834716797, "generatedTokens": 256, "loadSeconds": 9.059906005859375e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.07664299011230469, "prefillTokensPerSecond": 704.5654132344524, "promptTokens": 54, "residentMemoryAfterBytes": 9467527168, "residentMemoryBeforeBytes": 9467461632, "ttftSeconds": 0.07689309120178223 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/ornith-c35/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "8f2cd3cdbaad81517a5df24812258b42ba0d4933f3fecfa7080111270102c728", "receipt": { "after": { "gpuDeviceUtilizationPercent": 72, "loadAverage": [ 6.60400390625, 6.55078125, 7.37744140625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 6.79931640625, 6.568359375, 7.408203125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.09021329879760742, "loadAverage": [ 6.79931640625, 6.568359375, 7.408203125 ] }, { "deviceUtilizationPercent": 29, "elapsedSeconds": 2.1937642097473145, "loadAverage": [ 7.21533203125, 6.658203125, 7.43505859375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 4.302427291870117, "loadAverage": [ 7.21533203125, 6.658203125, 7.43505859375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 6.408513069152832, "loadAverage": [ 7.21533203125, 6.658203125, 7.43505859375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 8.53226923942566, "loadAverage": [ 7.03759765625, 6.63037109375, 7.42041015625 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 10.644622087478638, "loadAverage": [ 7.03759765625, 6.63037109375, 7.42041015625 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 12.743419170379639, "loadAverage": [ 6.7939453125, 6.58642578125, 7.39990234375 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 14.848475217819214, "loadAverage": [ 6.7939453125, 6.58642578125, 7.39990234375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 16.991605043411255, "loadAverage": [ 6.7939453125, 6.58642578125, 7.39990234375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 19.12017822265625, "loadAverage": [ 6.56982421875, 6.54345703125, 7.3798828125 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 21.241218090057373, "loadAverage": [ 6.56982421875, 6.54345703125, 7.3798828125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9462972416, "processSeconds": 23.360762119293213, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.35", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621879.1798248, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.1410732342359655 }, "endToEndSpeedups": { "adaptive": 1.1369387338834978 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.35135135135135137, "acceptedDraftTokens": 13, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 37, "rounds": 27, "route": "mtp-speculative" }, "decodeSeconds": 2.004284977912903, "decodeTokensPerSecond": 127.7263477105822, "elapsedSeconds": 2.082953929901123, "endToEndTokensPerSecond": 122.90238220110429, "expectedMTPActive": true, "firstTokenSeconds": 0.00035393238067626953, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.07696199417114258, "prefillTokensPerSecond": 779.605578651929, "promptTokens": 60, "residentMemoryAfterBytes": 9462595584, "residentMemoryBeforeBytes": 9462431744, "ttftSeconds": 0.07732200622558594 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.2870359420776367, "decodeTokensPerSecond": 111.93527626305652, "elapsedSeconds": 2.3681910037994385, "endToEndTokensPerSecond": 108.09938876943752, "expectedMTPActive": false, "firstTokenSeconds": 0.007764935493469238, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.07937800884246826, "prefillTokensPerSecond": 755.8768590312538, "promptTokens": 60, "residentMemoryAfterBytes": 9333374976, "residentMemoryBeforeBytes": 9333358592, "ttftSeconds": 0.08714890480041504 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0226831934141067 }, "endToEndSpeedups": { "adaptive": 1.0269078496183501 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.35135135135135137, "acceptedDraftTokens": 13, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 37, "rounds": 27, "route": "mtp-speculative" }, "decodeSeconds": 2.204231023788452, "decodeTokensPerSecond": 116.1402762401956, "elapsedSeconds": 2.2830159664154053, "endToEndTokensPerSecond": 112.1323739149968, "expectedMTPActive": true, "firstTokenSeconds": 0.00017702579498291016, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.07688391208648682, "prefillTokensPerSecond": 780.397333742668, "promptTokens": 60, "residentMemoryAfterBytes": 9462857728, "residentMemoryBeforeBytes": 9462693888, "ttftSeconds": 0.07706689834594727 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.25423002243042, "decodeTokensPerSecond": 113.56427580712953, "elapsedSeconds": 2.3444470167160034, "endToEndTokensPerSecond": 109.19419299080316, "expectedMTPActive": false, "firstTokenSeconds": 0.00899207592010498, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.08819401264190674, "prefillTokensPerSecond": 680.3182914878519, "promptTokens": 60, "residentMemoryAfterBytes": 9333391360, "residentMemoryBeforeBytes": 9333374976, "ttftSeconds": 0.09719407558441162 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9746427989289399 }, "endToEndSpeedups": { "adaptive": 0.9765837767275062 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.35135135135135137, "acceptedDraftTokens": 13, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 37, "rounds": 27, "route": "mtp-speculative" }, "decodeSeconds": 2.3322750329971313, "decodeTokensPerSecond": 109.76407000808248, "elapsedSeconds": 2.41703999042511, "endToEndTokensPerSecond": 105.91467291154527, "expectedMTPActive": true, "firstTokenSeconds": 0.00039398670196533203, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.08265900611877441, "prefillTokensPerSecond": 725.8737168190091, "promptTokens": 60, "residentMemoryAfterBytes": 9462972416, "residentMemoryBeforeBytes": 9462857728, "ttftSeconds": 0.08306097984313965 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.2731350660324097, "decodeTokensPerSecond": 112.6197927370982, "elapsedSeconds": 2.360442042350769, "endToEndTokensPerSecond": 108.45426212839739, "expectedMTPActive": false, "firstTokenSeconds": 0.00869607925415039, "generatedTokens": 256, "loadSeconds": 8.940696716308594e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.08537507057189941, "prefillTokensPerSecond": 702.7812638757404, "promptTokens": 60, "residentMemoryAfterBytes": 9333473280, "residentMemoryBeforeBytes": 9333391360, "ttftSeconds": 0.09408009052276611 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/ornith-current/ornith-code", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "cb6c69a11fa006a86c4298673d89b7f1737867ac2c5feeee2030c6607fbde952", "receipt": { "after": { "gpuDeviceUtilizationPercent": 67, "loadAverage": [ 6.4990234375, 6.76904296875, 7.63037109375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7284.25M free = 907.75M (encrypted)", "swapUsedMiB": 7284.25 }, "before": { "gpuDeviceUtilizationPercent": 33, "loadAverage": [ 5.826171875, 6.958984375, 7.78515625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7284.25M free = 907.75M (encrypted)", "swapUsedMiB": 7284.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08044600486755371, "loadAverage": [ 5.826171875, 6.958984375, 7.78515625 ] }, { "deviceUtilizationPercent": 0, "elapsedSeconds": 2.1741960048675537, "loadAverage": [ 5.826171875, 6.958984375, 7.78515625 ] }, { "deviceUtilizationPercent": 6, "elapsedSeconds": 4.268815040588379, "loadAverage": [ 5.439453125, 6.85986328125, 7.7451171875 ] }, { "deviceUtilizationPercent": 8, "elapsedSeconds": 6.354426860809326, "loadAverage": [ 5.439453125, 6.85986328125, 7.7451171875 ] }, { "deviceUtilizationPercent": 2, "elapsedSeconds": 8.446568965911865, "loadAverage": [ 5.16357421875, 6.77880859375, 7.71142578125 ] }, { "deviceUtilizationPercent": 2, "elapsedSeconds": 10.538522005081177, "loadAverage": [ 5.16357421875, 6.77880859375, 7.71142578125 ] }, { "deviceUtilizationPercent": 1, "elapsedSeconds": 12.627027034759521, "loadAverage": [ 5.16357421875, 6.77880859375, 7.71142578125 ] }, { "deviceUtilizationPercent": 21, "elapsedSeconds": 14.722172975540161, "loadAverage": [ 4.91015625, 6.69921875, 7.677734375 ] }, { "deviceUtilizationPercent": 7, "elapsedSeconds": 16.810291051864624, "loadAverage": [ 4.91015625, 6.69921875, 7.677734375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 18.90547800064087, "loadAverage": [ 4.5966796875, 6.6044921875, 7.63818359375 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 21.000041961669922, "loadAverage": [ 4.5966796875, 6.6044921875, 7.63818359375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 23.091065883636475, "loadAverage": [ 4.62890625, 6.57763671875, 7.62255859375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 25.18607997894287, "loadAverage": [ 4.62890625, 6.57763671875, 7.62255859375 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 27.282843828201294, "loadAverage": [ 4.62890625, 6.57763671875, 7.62255859375 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 29.378443956375122, "loadAverage": [ 4.8984375, 6.60107421875, 7.62451171875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 31.473056077957153, "loadAverage": [ 4.8984375, 6.60107421875, 7.62451171875 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 33.571044921875, "loadAverage": [ 4.666015625, 6.5244140625, 7.59130859375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 35.665650844573975, "loadAverage": [ 4.666015625, 6.5244140625, 7.59130859375 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 37.76602506637573, "loadAverage": [ 4.666015625, 6.5244140625, 7.59130859375 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 39.859127044677734, "loadAverage": [ 5.0126953125, 6.5654296875, 7.59912109375 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 41.95647692680359, "loadAverage": [ 5.0126953125, 6.5654296875, 7.59912109375 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 44.05706000328064, "loadAverage": [ 5.41162109375, 6.6220703125, 7.61279296875 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 46.1512770652771, "loadAverage": [ 5.41162109375, 6.6220703125, 7.61279296875 ] }, { "deviceUtilizationPercent": 47, "elapsedSeconds": 48.24592590332031, "loadAverage": [ 5.61865234375, 6.64453125, 7.61474609375 ] }, { "deviceUtilizationPercent": 34, "elapsedSeconds": 50.34685492515564, "loadAverage": [ 5.61865234375, 6.64453125, 7.61474609375 ] }, { "deviceUtilizationPercent": 14, "elapsedSeconds": 52.44521713256836, "loadAverage": [ 5.61865234375, 6.64453125, 7.61474609375 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 54.54545211791992, "loadAverage": [ 5.80908203125, 6.6669921875, 7.61669921875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 56.64780902862549, "loadAverage": [ 5.80908203125, 6.6669921875, 7.61669921875 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 58.74777913093567, "loadAverage": [ 6.064453125, 6.70556640625, 7.62451171875 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 60.8488450050354, "loadAverage": [ 6.064453125, 6.70556640625, 7.62451171875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 62.951130867004395, "loadAverage": [ 6.064453125, 6.70556640625, 7.62451171875 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 65.04809808731079, "loadAverage": [ 6.21923828125, 6.72705078125, 7.62646484375 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 67.14617896080017, "loadAverage": [ 6.21923828125, 6.72705078125, 7.62646484375 ] }, { "deviceUtilizationPercent": 45, "elapsedSeconds": 69.24268174171448, "loadAverage": [ 6.28173828125, 6.7314453125, 7.62255859375 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 71.34277892112732, "loadAverage": [ 6.28173828125, 6.7314453125, 7.62255859375 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 73.4442949295044, "loadAverage": [ 6.4990234375, 6.76904296875, 7.63037109375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9465626624, "processSeconds": 75.56408905982971, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "0", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "0", "MERERUN_Q35_SCOPED_COMPILE": "0" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621668.162966, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0145822943673193 }, "endToEndSpeedups": { "adaptive": 0.9962947392324731 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.753132939338684, "decodeTokensPerSecond": 92.98497589494986, "elapsedSeconds": 2.8493560552597046, "endToEndTokensPerSecond": 89.8448614477094, "expectedMTPActive": false, "firstTokenSeconds": 0.010051965713500977, "generatedTokens": 256, "loadSeconds": 5.0067901611328125e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.09485197067260742, "prefillTokensPerSecond": 569.308150553743, "promptTokens": 54, "residentMemoryAfterBytes": 7681687552, "residentMemoryBeforeBytes": 7681294336, "ttftSeconds": 0.10490894317626953 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 2.7135629653930664, "decodeTokensPerSecond": 94.34091018518811, "elapsedSeconds": 2.859952926635742, "endToEndTokensPerSecond": 89.51196280742332, "expectedMTPActive": true, "firstTokenSeconds": 0.0004099607467651367, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.14471793174743652, "prefillTokensPerSecond": 373.1396610493401, "promptTokens": 54, "residentMemoryAfterBytes": 9465331712, "residentMemoryBeforeBytes": 9465167872, "ttftSeconds": 0.14513397216796875 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.4587341633593276 }, "endToEndSpeedups": { "adaptive": 1.4522251372150823 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 11.044227957725525, "decodeTokensPerSecond": 23.179528798201414, "elapsedSeconds": 11.159536957740784, "endToEndTokensPerSecond": 22.940019910272913, "expectedMTPActive": false, "firstTokenSeconds": 0.035768985748291016, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.11391401290893555, "prefillTokensPerSecond": 474.0417673036271, "promptTokens": 54, "residentMemoryAfterBytes": 7681900544, "residentMemoryBeforeBytes": 7681785856, "ttftSeconds": 0.14968907833099365 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 7.571103930473328, "decodeTokensPerSecond": 33.81277054850778, "elapsedSeconds": 7.684440016746521, "endToEndTokensPerSecond": 33.3140735619128, "expectedMTPActive": true, "firstTokenSeconds": 0.0001779794692993164, "generatedTokens": 256, "loadSeconds": 6.9141387939453125e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.11170697212219238, "prefillTokensPerSecond": 483.40760629454064, "promptTokens": 54, "residentMemoryAfterBytes": 9465495552, "residentMemoryBeforeBytes": 9465331712, "ttftSeconds": 0.11189186573028564 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 5.684248955463109 }, "endToEndSpeedups": { "adaptive": 5.457757220444022 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 14.967503070831299, "decodeTokensPerSecond": 17.103721227817438, "elapsedSeconds": 15.104926943778992, "endToEndTokensPerSecond": 16.948112424034885, "expectedMTPActive": false, "firstTokenSeconds": 0.0425419807434082, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.1359950304031372, "prefillTokensPerSecond": 397.0733330469868, "promptTokens": 54, "residentMemoryAfterBytes": 7682375680, "residentMemoryBeforeBytes": 7681900544, "ttftSeconds": 0.1785430908203125 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 2.633154034614563, "decodeTokensPerSecond": 97.22180952375348, "elapsedSeconds": 2.7676069736480713, "endToEndTokensPerSecond": 92.49868295517344, "expectedMTPActive": true, "firstTokenSeconds": 0.00036895275115966797, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.13273298740386963, "prefillTokensPerSecond": 406.8317986070259, "promptTokens": 54, "residentMemoryAfterBytes": 9465626624, "residentMemoryBeforeBytes": 9465495552, "ttftSeconds": 0.1331080198287964 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/ornith-current/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "bc1063a3b1a006b78ee077b691e43bc3601831c815fad800e21da1135a03db88", "receipt": { "after": { "gpuDeviceUtilizationPercent": 75, "loadAverage": [ 5.447265625, 6.4443359375, 7.44091796875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 6.4990234375, 6.76904296875, 7.63037109375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7284.25M free = 907.75M (encrypted)", "swapUsedMiB": 7284.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "4" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08449792861938477, "loadAverage": [ 6.4990234375, 6.76904296875, 7.63037109375 ] }, { "deviceUtilizationPercent": 36, "elapsedSeconds": 2.1868579387664795, "loadAverage": [ 6.4990234375, 6.76904296875, 7.63037109375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 4.288841962814331, "loadAverage": [ 6.21875, 6.7060546875, 7.60302734375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 6.388010025024414, "loadAverage": [ 6.21875, 6.7060546875, 7.60302734375 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 8.487811088562012, "loadAverage": [ 6.201171875, 6.69384765625, 7.59326171875 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 10.580504179000854, "loadAverage": [ 6.201171875, 6.69384765625, 7.59326171875 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 12.67274808883667, "loadAverage": [ 6.02490234375, 6.64892578125, 7.57177734375 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 14.773230075836182, "loadAverage": [ 6.02490234375, 6.64892578125, 7.57177734375 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 16.867496967315674, "loadAverage": [ 6.02490234375, 6.64892578125, 7.57177734375 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 18.964004278182983, "loadAverage": [ 6.1826171875, 6.6708984375, 7.57421875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 21.058720111846924, "loadAverage": [ 6.1826171875, 6.6708984375, 7.57421875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 23.157839059829712, "loadAverage": [ 6.328125, 6.69287109375, 7.57666015625 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 25.25394606590271, "loadAverage": [ 6.328125, 6.69287109375, 7.57666015625 ] }, { "deviceUtilizationPercent": 47, "elapsedSeconds": 27.357171058654785, "loadAverage": [ 6.328125, 6.69287109375, 7.57666015625 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 29.45439910888672, "loadAverage": [ 6.5419921875, 6.73095703125, 7.5849609375 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 31.550530195236206, "loadAverage": [ 6.5419921875, 6.73095703125, 7.5849609375 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 33.64287304878235, "loadAverage": [ 6.49853515625, 6.71875, 7.5751953125 ] }, { "deviceUtilizationPercent": 31, "elapsedSeconds": 35.74376130104065, "loadAverage": [ 6.49853515625, 6.71875, 7.5751953125 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 37.84064221382141, "loadAverage": [ 6.45849609375, 6.70654296875, 7.56591796875 ] }, { "deviceUtilizationPercent": 45, "elapsedSeconds": 39.935879945755005, "loadAverage": [ 6.45849609375, 6.70654296875, 7.56591796875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 42.03589415550232, "loadAverage": [ 6.45849609375, 6.70654296875, 7.56591796875 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 44.12905025482178, "loadAverage": [ 6.26123046875, 6.6611328125, 7.544921875 ] }, { "deviceUtilizationPercent": 0, "elapsedSeconds": 46.220110177993774, "loadAverage": [ 6.26123046875, 6.6611328125, 7.544921875 ] }, { "deviceUtilizationPercent": 45, "elapsedSeconds": 48.31564116477966, "loadAverage": [ 6.0, 6.60009765625, 7.51806640625 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 50.41105103492737, "loadAverage": [ 6.0, 6.60009765625, 7.51806640625 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 52.505523920059204, "loadAverage": [ 6.16015625, 6.623046875, 7.5205078125 ] }, { "deviceUtilizationPercent": 47, "elapsedSeconds": 54.60018825531006, "loadAverage": [ 6.16015625, 6.623046875, 7.5205078125 ] }, { "deviceUtilizationPercent": 47, "elapsedSeconds": 56.69531607627869, "loadAverage": [ 6.16015625, 6.623046875, 7.5205078125 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 58.78992509841919, "loadAverage": [ 5.98681640625, 6.5791015625, 7.49951171875 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 60.889541149139404, "loadAverage": [ 5.98681640625, 6.5791015625, 7.49951171875 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 62.98503518104553, "loadAverage": [ 5.74755859375, 6.51953125, 7.47314453125 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 65.0788049697876, "loadAverage": [ 5.74755859375, 6.51953125, 7.47314453125 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 67.17199516296387, "loadAverage": [ 5.74755859375, 6.51953125, 7.47314453125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9462726656, "processSeconds": 69.27412724494934, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "0", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "0", "MERERUN_Q35_SCOPED_COMPILE": "0" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621743.847849, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.24795829896450483 }, "endToEndSpeedups": { "adaptive": 0.2581012202123395 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 11.13976001739502, "decodeTokensPerSecond": 22.98074640748539, "elapsedSeconds": 11.256517887115479, "endToEndTokensPerSecond": 22.742379354544862, "expectedMTPActive": true, "firstTokenSeconds": 0.0001989603042602539, "generatedTokens": 256, "loadSeconds": 6.079673767089844e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.11489701271057129, "prefillTokensPerSecond": 522.2067883622148, "promptTokens": 60, "residentMemoryAfterBytes": 9462022144, "residentMemoryBeforeBytes": 9455878144, "ttftSeconds": 0.11510205268859863 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.7621959447860718, "decodeTokensPerSecond": 92.679884091216, "elapsedSeconds": 2.9053210020065308, "endToEndTokensPerSecond": 88.11418766573338, "expectedMTPActive": false, "firstTokenSeconds": 0.011691927909851074, "generatedTokens": 256, "loadSeconds": 6.9141387939453125e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.1413710117340088, "prefillTokensPerSecond": 424.4151560073058, "promptTokens": 60, "residentMemoryAfterBytes": 9340862464, "residentMemoryBeforeBytes": 9340780544, "ttftSeconds": 0.1530698537826538 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 3.9744117402503765 }, "endToEndSpeedups": { "adaptive": 3.819465327026811 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 3.4193450212478638, "decodeTokensPerSecond": 74.86813948554824, "elapsedSeconds": 3.584097981452942, "endToEndTokensPerSecond": 71.42661872659555, "expectedMTPActive": true, "firstTokenSeconds": 0.0004349946975708008, "generatedTokens": 256, "loadSeconds": 5.0067901611328125e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.16295599937438965, "prefillTokensPerSecond": 368.19755167252634, "promptTokens": 60, "residentMemoryAfterBytes": 9462349824, "residentMemoryBeforeBytes": 9462120448, "ttftSeconds": 0.16339600086212158 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 13.589884996414185, "decodeTokensPerSecond": 18.837539836985226, "elapsedSeconds": 13.689337968826294, "endToEndTokensPerSecond": 18.70068520354817, "expectedMTPActive": false, "firstTokenSeconds": 0.04110395908355713, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.09785699844360352, "prefillTokensPerSecond": 613.1395909775315, "promptTokens": 60, "residentMemoryAfterBytes": 9340911616, "residentMemoryBeforeBytes": 9340862464, "ttftSeconds": 0.13896691799163818 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.20702898008284243 }, "endToEndSpeedups": { "adaptive": 0.2138995125701697 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 13.427946090698242, "decodeTokensPerSecond": 19.0647175875494, "elapsedSeconds": 13.545103073120117, "endToEndTokensPerSecond": 18.899819264426636, "expectedMTPActive": true, "firstTokenSeconds": 0.00017702579498291016, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.1153249740600586, "prefillTokensPerSecond": 520.2689225731226, "promptTokens": 60, "residentMemoryAfterBytes": 9462743040, "residentMemoryBeforeBytes": 9462349824, "ttftSeconds": 0.11550796031951904 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.7799739837646484, "decodeTokensPerSecond": 92.08719272017218, "elapsedSeconds": 2.8972909450531006, "endToEndTokensPerSecond": 88.35840267857812, "expectedMTPActive": false, "firstTokenSeconds": 0.010432004928588867, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.11570405960083008, "prefillTokensPerSecond": 518.5643460306864, "promptTokens": 60, "residentMemoryAfterBytes": 9340977152, "residentMemoryBeforeBytes": 9340911616, "ttftSeconds": 0.12614202499389648 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-c18/qwen-code", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "ae3ad0408b23796a01790a8d88620cf41f2e8141a7ce0b7ac5f815b420502564", "receipt": { "after": { "gpuDeviceUtilizationPercent": 64, "loadAverage": [ 16.2158203125, 13.74462890625, 10.48193359375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 21, "loadAverage": [ 12.73046875, 10.03955078125, 8.75439453125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 25, "elapsedSeconds": 0.08994936943054199, "loadAverage": [ 12.73046875, 10.03955078125, 8.75439453125 ] }, { "deviceUtilizationPercent": 16, "elapsedSeconds": 2.2729241847991943, "loadAverage": [ 12.73046875, 10.03955078125, 8.75439453125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 4.51036524772644, "loadAverage": [ 15.955078125, 10.75244140625, 9.01318359375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 6.744260311126709, "loadAverage": [ 15.955078125, 10.75244140625, 9.01318359375 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 8.985917329788208, "loadAverage": [ 15.955078125, 10.75244140625, 9.01318359375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 11.202836275100708, "loadAverage": [ 18.04052734375, 11.27099609375, 9.2060546875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 13.442066192626953, "loadAverage": [ 18.04052734375, 11.27099609375, 9.2060546875 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 15.679009199142456, "loadAverage": [ 20.439453125, 11.88037109375, 9.43310546875 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 17.893596172332764, "loadAverage": [ 20.439453125, 11.88037109375, 9.43310546875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 20.13033437728882, "loadAverage": [ 22.32568359375, 12.41357421875, 9.63525390625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 22.384579181671143, "loadAverage": [ 22.32568359375, 12.41357421875, 9.63525390625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 24.846886157989502, "loadAverage": [ 22.53955078125, 12.6220703125, 9.72509765625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 27.331428289413452, "loadAverage": [ 22.53955078125, 12.6220703125, 9.72509765625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 29.856426000595093, "loadAverage": [ 21.935546875, 12.6611328125, 9.755859375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 32.254027128219604, "loadAverage": [ 21.935546875, 12.6611328125, 9.755859375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 34.57110118865967, "loadAverage": [ 22.1005859375, 12.84912109375, 9.8388671875 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 37.10563516616821, "loadAverage": [ 22.1005859375, 12.84912109375, 9.8388671875 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 39.58717226982117, "loadAverage": [ 20.89111328125, 12.751953125, 9.82177734375 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 41.97731423377991, "loadAverage": [ 20.89111328125, 12.751953125, 9.82177734375 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 44.56009340286255, "loadAverage": [ 19.8583984375, 12.6728515625, 9.81103515625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 47.05228519439697, "loadAverage": [ 19.8583984375, 12.6728515625, 9.81103515625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 49.3834912776947, "loadAverage": [ 19.8583984375, 12.6728515625, 9.81103515625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 51.65529704093933, "loadAverage": [ 18.50830078125, 12.51220703125, 9.77099609375 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 53.99842619895935, "loadAverage": [ 18.50830078125, 12.51220703125, 9.77099609375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 56.274572134017944, "loadAverage": [ 19.50830078125, 12.81884765625, 9.89501953125 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 58.69168734550476, "loadAverage": [ 19.50830078125, 12.81884765625, 9.89501953125 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 61.02362012863159, "loadAverage": [ 22.75048828125, 13.60205078125, 10.1884765625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 63.79757118225098, "loadAverage": [ 22.75048828125, 13.60205078125, 10.1884765625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 66.57898807525635, "loadAverage": [ 25.41259765625, 14.3056640625, 10.45654296875 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 69.18661832809448, "loadAverage": [ 25.41259765625, 14.3056640625, 10.45654296875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 72.06473422050476, "loadAverage": [ 25.05908203125, 14.41650390625, 10.51806640625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 74.69224500656128, "loadAverage": [ 27.05615234375, 15.0068359375, 10.7490234375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 77.41719031333923, "loadAverage": [ 27.05615234375, 15.0068359375, 10.7490234375 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 79.86537218093872, "loadAverage": [ 25.85009765625, 14.95654296875, 10.75634765625 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 82.01041007041931, "loadAverage": [ 25.85009765625, 14.95654296875, 10.75634765625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 84.12613916397095, "loadAverage": [ 25.85009765625, 14.95654296875, 10.75634765625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 86.2422924041748, "loadAverage": [ 24.26025390625, 14.8076171875, 10.72802734375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 88.3397102355957, "loadAverage": [ 24.26025390625, 14.8076171875, 10.72802734375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 90.43194031715393, "loadAverage": [ 22.6376953125, 14.6279296875, 10.6884765625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 92.53108620643616, "loadAverage": [ 22.6376953125, 14.6279296875, 10.6884765625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 94.62742829322815, "loadAverage": [ 20.90478515625, 14.4013671875, 10.63134765625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 96.72533130645752, "loadAverage": [ 20.90478515625, 14.4013671875, 10.63134765625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 98.83446025848389, "loadAverage": [ 20.90478515625, 14.4013671875, 10.63134765625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 100.93036532402039, "loadAverage": [ 19.470703125, 14.2119140625, 10.58642578125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 103.02559733390808, "loadAverage": [ 19.470703125, 14.2119140625, 10.58642578125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 105.11733317375183, "loadAverage": [ 18.1513671875, 14.025390625, 10.54150390625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 107.21281027793884, "loadAverage": [ 18.1513671875, 14.025390625, 10.54150390625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 109.31486535072327, "loadAverage": [ 18.1513671875, 14.025390625, 10.54150390625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 111.4118402004242, "loadAverage": [ 17.01806640625, 13.85888671875, 10.5029296875 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 113.5851502418518, "loadAverage": [ 17.01806640625, 13.85888671875, 10.5029296875 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15825534976, "processSeconds": 115.73394727706909, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788622152.0609658, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.268967813829826 }, "endToEndSpeedups": { "adaptive": 2.1998864014325235 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.311511993408203, "decodeTokensPerSecond": 15.69443716213767, "elapsedSeconds": 17.112749099731445, "endToEndTokensPerSecond": 14.95960692861543, "expectedMTPActive": false, "firstTokenSeconds": 0.18932998180389404, "generatedTokens": 256, "loadSeconds": 4.506111145019531e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.7973200082778931, "prefillTokensPerSecond": 67.7268843618172, "promptTokens": 54, "residentMemoryAfterBytes": 15408726016, "residentMemoryBeforeBytes": 15408168960, "ttftSeconds": 0.9866950511932373 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 7.18895697593689, "decodeTokensPerSecond": 35.61017277706509, "elapsedSeconds": 7.778923988342285, "endToEndTokensPerSecond": 32.90943585303685, "expectedMTPActive": true, "firstTokenSeconds": 0.0004049539566040039, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.5884289741516113, "prefillTokensPerSecond": 91.76978424262069, "promptTokens": 54, "residentMemoryAfterBytes": 15825453056, "residentMemoryBeforeBytes": 15825289216, "ttftSeconds": 0.5888539552688599 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.1500023550409892 }, "endToEndSpeedups": { "adaptive": 2.0355788658270653 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 15.818511009216309, "decodeTokensPerSecond": 16.18357125084954, "elapsedSeconds": 16.30987799167633, "endToEndTokensPerSecond": 15.696009506058132, "expectedMTPActive": false, "firstTokenSeconds": 0.05109703540802002, "generatedTokens": 256, "loadSeconds": 6.103515625e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.4870070219039917, "prefillTokensPerSecond": 110.88135811447403, "promptTokens": 54, "residentMemoryAfterBytes": 15408939008, "residentMemoryBeforeBytes": 15408840704, "ttftSeconds": 0.5381650924682617 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 7.357438921928406, "decodeTokensPerSecond": 34.79471630230016, "elapsedSeconds": 8.012402892112732, "endToEndTokensPerSecond": 31.950465228352645, "expectedMTPActive": true, "firstTokenSeconds": 0.0003019571304321289, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.6535149812698364, "prefillTokensPerSecond": 82.63008737010635, "promptTokens": 54, "residentMemoryAfterBytes": 15825485824, "residentMemoryBeforeBytes": 15825453056, "ttftSeconds": 0.6538349390029907 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.8682243538654957 }, "endToEndSpeedups": { "adaptive": 2.7103948283552017 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.767886996269226, "decodeTokensPerSecond": 11.760443264147575, "elapsedSeconds": 22.133236050605774, "endToEndTokensPerSecond": 11.566315897714986, "expectedMTPActive": false, "firstTokenSeconds": 0.0465700626373291, "generatedTokens": 256, "loadSeconds": 4.601478576660156e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.3599209785461426, "prefillTokensPerSecond": 150.03293283466414, "promptTokens": 54, "residentMemoryAfterBytes": 15409004544, "residentMemoryBeforeBytes": 15408939008, "ttftSeconds": 0.4065370559692383 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 7.5893250703811646, "decodeTokensPerSecond": 33.7315897824815, "elapsedSeconds": 8.166056036949158, "endToEndTokensPerSecond": 31.34928279228925, "expectedMTPActive": true, "firstTokenSeconds": 0.0004380941390991211, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.574834942817688, "prefillTokensPerSecond": 93.94000951874352, "promptTokens": 54, "residentMemoryAfterBytes": 15825534976, "residentMemoryBeforeBytes": 15825485824, "ttftSeconds": 0.5752910375595093 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-c18/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "aa62f4ccd6741fb5e2a88d1cbb62f48ca829c6a975e937d2dca27643db253596", "receipt": { "after": { "gpuDeviceUtilizationPercent": 92, "loadAverage": [ 8.36279296875, 11.66259765625, 10.0908203125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 41, "loadAverage": [ 16.2158203125, 13.74462890625, 10.48193359375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 39, "elapsedSeconds": 0.11649203300476074, "loadAverage": [ 16.2158203125, 13.74462890625, 10.48193359375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 2.2170190811157227, "loadAverage": [ 16.2158203125, 13.74462890625, 10.48193359375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 4.311152935028076, "loadAverage": [ 14.9970703125, 13.53271484375, 10.42626953125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 6.403930902481079, "loadAverage": [ 14.9970703125, 13.53271484375, 10.42626953125 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 8.500807046890259, "loadAverage": [ 14.9970703125, 13.53271484375, 10.42626953125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 10.597621202468872, "loadAverage": [ 14.0361328125, 13.357421875, 10.38232421875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 12.692683219909668, "loadAverage": [ 14.0361328125, 13.357421875, 10.38232421875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 14.789412021636963, "loadAverage": [ 13.072265625, 13.16845703125, 10.3330078125 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 16.88393211364746, "loadAverage": [ 13.072265625, 13.16845703125, 10.3330078125 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 18.976946115493774, "loadAverage": [ 12.42578125, 13.03271484375, 10.3017578125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 21.066922187805176, "loadAverage": [ 12.42578125, 13.03271484375, 10.3017578125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 23.160953044891357, "loadAverage": [ 12.42578125, 13.03271484375, 10.3017578125 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 25.25238013267517, "loadAverage": [ 11.7509765625, 12.88232421875, 10.2646484375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 27.349311113357544, "loadAverage": [ 11.7509765625, 12.88232421875, 10.2646484375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 29.44742703437805, "loadAverage": [ 11.2099609375, 12.75146484375, 10.2333984375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 31.551965951919556, "loadAverage": [ 11.2099609375, 12.75146484375, 10.2333984375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 33.645081996917725, "loadAverage": [ 10.63232421875, 12.60595703125, 10.19677734375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 35.73726201057434, "loadAverage": [ 10.63232421875, 12.60595703125, 10.19677734375 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 37.8272819519043, "loadAverage": [ 10.63232421875, 12.60595703125, 10.19677734375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 39.920876264572144, "loadAverage": [ 10.02099609375, 12.4462890625, 10.154296875 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 42.01561617851257, "loadAverage": [ 10.02099609375, 12.4462890625, 10.154296875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 44.11212396621704, "loadAverage": [ 9.61865234375, 12.322265625, 10.1240234375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 46.20360612869263, "loadAverage": [ 9.61865234375, 12.322265625, 10.1240234375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 48.296584129333496, "loadAverage": [ 9.61865234375, 12.322265625, 10.1240234375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 50.390113830566406, "loadAverage": [ 9.00830078125, 12.15087890625, 10.076171875 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 52.4989550113678, "loadAverage": [ 9.00830078125, 12.15087890625, 10.076171875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 54.628763914108276, "loadAverage": [ 8.60693359375, 12.01513671875, 10.04052734375 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 57.129313230514526, "loadAverage": [ 8.60693359375, 12.01513671875, 10.04052734375 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 59.608875036239624, "loadAverage": [ 11.36083984375, 12.529296875, 10.2333984375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 61.98652982711792, "loadAverage": [ 11.36083984375, 12.529296875, 10.2333984375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 64.49066996574402, "loadAverage": [ 11.89208984375, 12.6201171875, 10.27880859375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 67.03268599510193, "loadAverage": [ 11.89208984375, 12.6201171875, 10.27880859375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 69.91038012504578, "loadAverage": [ 12.30078125, 12.6923828125, 10.31787109375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 72.6122350692749, "loadAverage": [ 12.30078125, 12.6923828125, 10.31787109375 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 75.35951590538025, "loadAverage": [ 13.8779296875, 13.0126953125, 10.44482421875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 77.61324191093445, "loadAverage": [ 13.8779296875, 13.0126953125, 10.44482421875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 79.76711988449097, "loadAverage": [ 13.4072265625, 12.92919921875, 10.43017578125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 81.90829086303711, "loadAverage": [ 13.4072265625, 12.92919921875, 10.43017578125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 84.0415608882904, "loadAverage": [ 12.57373046875, 12.76416015625, 10.38623046875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 86.16651821136475, "loadAverage": [ 12.57373046875, 12.76416015625, 10.38623046875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 88.28314900398254, "loadAverage": [ 12.57373046875, 12.76416015625, 10.38623046875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 90.38642501831055, "loadAverage": [ 11.88671875, 12.61865234375, 10.3486328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 92.50648093223572, "loadAverage": [ 11.88671875, 12.61865234375, 10.3486328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 94.62218403816223, "loadAverage": [ 11.2548828125, 12.47509765625, 10.31103515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 96.72173500061035, "loadAverage": [ 11.2548828125, 12.47509765625, 10.31103515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 98.85145497322083, "loadAverage": [ 10.75390625, 12.3505859375, 10.27978515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 100.98022103309631, "loadAverage": [ 10.75390625, 12.3505859375, 10.27978515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 103.10233807563782, "loadAverage": [ 10.75390625, 12.3505859375, 10.27978515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 105.2264609336853, "loadAverage": [ 10.29296875, 12.228515625, 10.24853515625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 107.33183598518372, "loadAverage": [ 10.29296875, 12.228515625, 10.24853515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 109.4474241733551, "loadAverage": [ 9.70849609375, 12.0751953125, 10.2060546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 111.55328702926636, "loadAverage": [ 9.70849609375, 12.0751953125, 10.2060546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 113.67311596870422, "loadAverage": [ 9.2509765625, 11.94091796875, 10.16943359375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 115.77610206604004, "loadAverage": [ 9.2509765625, 11.94091796875, 10.16943359375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 117.8837080001831, "loadAverage": [ 9.2509765625, 11.94091796875, 10.16943359375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 119.99038290977478, "loadAverage": [ 8.830078125, 11.80908203125, 10.1328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 122.10567688941956, "loadAverage": [ 8.830078125, 11.80908203125, 10.1328125 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15805136896, "processSeconds": 124.23306202888489, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788622267.974062, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.5080370540050727 }, "endToEndSpeedups": { "adaptive": 1.4616616499015473 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 11.698272943496704, "decodeTokensPerSecond": 21.88357215090586, "elapsedSeconds": 12.399502992630005, "endToEndTokensPerSecond": 20.645988807145, "expectedMTPActive": true, "firstTokenSeconds": 0.0004019737243652344, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.6994040012359619, "prefillTokensPerSecond": 85.78732734438198, "promptTokens": 60, "residentMemoryAfterBytes": 15804776448, "residentMemoryBeforeBytes": 15804334080, "ttftSeconds": 0.6998239755630493 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 17.64142906665802, "decodeTokensPerSecond": 14.511296053891424, "elapsedSeconds": 18.123878002166748, "endToEndTokensPerSecond": 14.125012316315232, "expectedMTPActive": false, "firstTokenSeconds": 0.15963101387023926, "generatedTokens": 256, "loadSeconds": 9.202957153320312e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.47853004932403564, "prefillTokensPerSecond": 125.38397554083615, "promptTokens": 60, "residentMemoryAfterBytes": 15418966016, "residentMemoryBeforeBytes": 15418867712, "ttftSeconds": 0.6382530927658081 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.4328934240681084 }, "endToEndSpeedups": { "adaptive": 1.4252402925136542 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 11.7376629114151, "decodeTokensPerSecond": 21.810133919507532, "elapsedSeconds": 12.467804908752441, "endToEndTokensPerSecond": 20.532884647584368, "expectedMTPActive": true, "firstTokenSeconds": 0.00029098987579345703, "generatedTokens": 256, "loadSeconds": 4.208087921142578e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.7281429767608643, "prefillTokensPerSecond": 82.40139905888994, "promptTokens": 60, "residentMemoryAfterBytes": 15805054976, "residentMemoryBeforeBytes": 15804891136, "ttftSeconds": 0.7284760475158691 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.818819999694824, "decodeTokensPerSecond": 15.221044044983245, "elapsedSeconds": 17.769617915153503, "endToEndTokensPerSecond": 14.406612523823, "expectedMTPActive": false, "firstTokenSeconds": 0.051451921463012695, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.9476369619369507, "prefillTokensPerSecond": 63.31538596527643, "promptTokens": 60, "residentMemoryAfterBytes": 15418949632, "residentMemoryBeforeBytes": 15418982400, "ttftSeconds": 0.9991148710250854 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.3600837881207897 }, "endToEndSpeedups": { "adaptive": 1.352929165886891 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 11.951074004173279, "decodeTokensPerSecond": 21.420668963358906, "elapsedSeconds": 12.69511103630066, "endToEndTokensPerSecond": 20.165243082001282, "expectedMTPActive": true, "firstTokenSeconds": 0.0002930164337158203, "generatedTokens": 256, "loadSeconds": 2.193450927734375e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.7422250509262085, "prefillTokensPerSecond": 80.83801526925983, "promptTokens": 60, "residentMemoryAfterBytes": 15805153280, "residentMemoryBeforeBytes": 15805054976, "ttftSeconds": 0.7425400018692017 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.254462003707886, "decodeTokensPerSecond": 15.749521573928597, "elapsedSeconds": 17.175585985183716, "endToEndTokensPerSecond": 14.90487720307388, "expectedMTPActive": false, "firstTokenSeconds": 0.05259203910827637, "generatedTokens": 256, "loadSeconds": 4.303455352783203e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.9180610179901123, "prefillTokensPerSecond": 65.35513307313329, "promptTokens": 60, "residentMemoryAfterBytes": 15419015168, "residentMemoryBeforeBytes": 15418949632, "ttftSeconds": 0.9706960916519165 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-c30/qwen-code", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "981120fcd69a9d240c110a081e66d94d62a5f3f579f6470eab225cae7c98b5fa", "receipt": { "after": { "gpuDeviceUtilizationPercent": 69, "loadAverage": [ 9.93896484375, 10.96044921875, 9.97705078125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 41, "loadAverage": [ 8.36279296875, 11.66259765625, 10.0908203125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 49, "elapsedSeconds": 0.08810806274414062, "loadAverage": [ 8.36279296875, 11.66259765625, 10.0908203125 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 2.19541597366333, "loadAverage": [ 8.36279296875, 11.66259765625, 10.0908203125 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 4.3029890060424805, "loadAverage": [ 7.93310546875, 11.5185546875, 10.048828125 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 6.413963079452515, "loadAverage": [ 7.93310546875, 11.5185546875, 10.048828125 ] }, { "deviceUtilizationPercent": 39, "elapsedSeconds": 8.514007806777954, "loadAverage": [ 7.93310546875, 11.5185546875, 10.048828125 ] }, { "deviceUtilizationPercent": 41, "elapsedSeconds": 10.628266096115112, "loadAverage": [ 7.69775390625, 11.41015625, 10.01904296875 ] }, { "deviceUtilizationPercent": 36, "elapsedSeconds": 12.73709511756897, "loadAverage": [ 7.69775390625, 11.41015625, 10.01904296875 ] }, { "deviceUtilizationPercent": 40, "elapsedSeconds": 14.840160131454468, "loadAverage": [ 7.6416015625, 11.3369140625, 10.0009765625 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 16.94929814338684, "loadAverage": [ 7.6416015625, 11.3369140625, 10.0009765625 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 19.053713083267212, "loadAverage": [ 7.6416015625, 11.3369140625, 10.0009765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 21.17399311065674, "loadAverage": [ 7.509765625, 11.248046875, 9.97705078125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 23.278244972229004, "loadAverage": [ 7.509765625, 11.248046875, 9.97705078125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 25.37913703918457, "loadAverage": [ 7.1484375, 11.11083984375, 9.93603515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 27.487257957458496, "loadAverage": [ 7.1484375, 11.11083984375, 9.93603515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 29.602432012557983, "loadAverage": [ 6.57568359375, 10.92626953125, 9.87744140625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 31.70416498184204, "loadAverage": [ 6.57568359375, 10.92626953125, 9.87744140625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 33.81872200965881, "loadAverage": [ 6.57568359375, 10.92626953125, 9.87744140625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 35.92046403884888, "loadAverage": [ 6.369140625, 10.81103515625, 9.8427734375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 38.025678873062134, "loadAverage": [ 6.369140625, 10.81103515625, 9.8427734375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 40.13490319252014, "loadAverage": [ 6.57958984375, 10.78076171875, 9.83740234375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 42.23935604095459, "loadAverage": [ 6.57958984375, 10.78076171875, 9.83740234375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 44.34653401374817, "loadAverage": [ 6.37255859375, 10.66796875, 9.802734375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 46.4538471698761, "loadAverage": [ 6.37255859375, 10.66796875, 9.802734375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 48.56352114677429, "loadAverage": [ 6.37255859375, 10.66796875, 9.802734375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 50.67429995536804, "loadAverage": [ 6.10205078125, 10.54052734375, 9.7626953125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 52.79206204414368, "loadAverage": [ 6.10205078125, 10.54052734375, 9.7626953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 54.97821402549744, "loadAverage": [ 5.93359375, 10.431640625, 9.728515625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 57.57572889328003, "loadAverage": [ 5.93359375, 10.431640625, 9.728515625 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 60.34495496749878, "loadAverage": [ 6.6591796875, 10.50732421875, 9.75927734375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 63.06168603897095, "loadAverage": [ 6.6591796875, 10.50732421875, 9.75927734375 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 65.80584716796875, "loadAverage": [ 7.56689453125, 10.63134765625, 9.80712890625 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 68.58082103729248, "loadAverage": [ 7.56689453125, 10.63134765625, 9.80712890625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 71.54237818717957, "loadAverage": [ 7.921875, 10.65380859375, 9.81982421875 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 73.9482901096344, "loadAverage": [ 7.921875, 10.65380859375, 9.81982421875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 76.19201707839966, "loadAverage": [ 8.48828125, 10.7255859375, 9.85009765625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 78.90555715560913, "loadAverage": [ 8.48828125, 10.7255859375, 9.85009765625 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 81.59321999549866, "loadAverage": [ 9.48974609375, 10.89599609375, 9.9150390625 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 84.24869298934937, "loadAverage": [ 9.48974609375, 10.89599609375, 9.9150390625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 86.85451984405518, "loadAverage": [ 10.4912109375, 11.080078125, 9.98583984375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 89.53718113899231, "loadAverage": [ 12.29345703125, 11.44384765625, 10.12060546875 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 91.72378301620483, "loadAverage": [ 12.29345703125, 11.44384765625, 10.12060546875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 93.82181215286255, "loadAverage": [ 12.29345703125, 11.44384765625, 10.12060546875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 95.9191062450409, "loadAverage": [ 11.708984375, 11.33642578125, 10.09033203125 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 98.01730394363403, "loadAverage": [ 11.708984375, 11.33642578125, 10.09033203125 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 100.11240792274475, "loadAverage": [ 10.93115234375, 11.18115234375, 10.04248046875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 102.21092104911804, "loadAverage": [ 10.93115234375, 11.18115234375, 10.04248046875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 104.30623602867126, "loadAverage": [ 10.4560546875, 11.078125, 10.0126953125 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 106.39893889427185, "loadAverage": [ 10.4560546875, 11.078125, 10.0126953125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 108.49073600769043, "loadAverage": [ 10.4560546875, 11.078125, 10.0126953125 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15817195520, "processSeconds": 110.60778713226318, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.3", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788622392.333998, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.9044892694489648 }, "endToEndSpeedups": { "adaptive": 1.9206018874168254 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.171544075012207, "decodeTokensPerSecond": 15.83027562566296, "elapsedSeconds": 17.096933007240295, "endToEndTokensPerSecond": 14.973445815783908, "expectedMTPActive": false, "firstTokenSeconds": 0.05190408229827881, "generatedTokens": 256, "loadSeconds": 2.09808349609375e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.922996997833252, "prefillTokensPerSecond": 58.50506570093482, "promptTokens": 54, "residentMemoryAfterBytes": 15396716544, "residentMemoryBeforeBytes": 15396569088, "ttftSeconds": 0.9749220609664917 }, { "acceleration": { "acceptanceRate": 0.7710843373493976, "acceptedDraftTokens": 192, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 249, "rounds": 64, "route": "mtp-speculative" }, "decodeSeconds": 8.491276025772095, "decodeTokensPerSecond": 30.148590061494605, "elapsedSeconds": 8.901862025260925, "endToEndTokensPerSecond": 28.75802829492814, "expectedMTPActive": true, "firstTokenSeconds": 0.01044309139251709, "generatedTokens": 256, "loadSeconds": 4.601478576660156e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.4070429801940918, "prefillTokensPerSecond": 132.6641230227112, "promptTokens": 54, "residentMemoryAfterBytes": 15817129984, "residentMemoryBeforeBytes": 15815376896, "ttftSeconds": 0.4175320863723755 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.286526283353526 }, "endToEndSpeedups": { "adaptive": 2.22807127253675 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 15.81271493434906, "decodeTokensPerSecond": 16.18950326132205, "elapsedSeconds": 16.6581369638443, "endToEndTokensPerSecond": 15.367864999287491, "expectedMTPActive": false, "firstTokenSeconds": 0.04707193374633789, "generatedTokens": 256, "loadSeconds": 5.1975250244140625e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.8409600257873535, "prefillTokensPerSecond": 64.21232679810458, "promptTokens": 54, "residentMemoryAfterBytes": 15396847616, "residentMemoryBeforeBytes": 15396831232, "ttftSeconds": 0.8880839347839355 }, { "acceleration": { "acceptanceRate": 0.7710843373493976, "acceptedDraftTokens": 192, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 249, "rounds": 64, "route": "mtp-speculative" }, "decodeSeconds": 6.9156060218811035, "decodeTokensPerSecond": 37.01772472145049, "elapsedSeconds": 7.47648298740387, "endToEndTokensPerSecond": 34.24069852513546, "expectedMTPActive": true, "firstTokenSeconds": 0.0003770589828491211, "generatedTokens": 256, "loadSeconds": 2.09808349609375e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.559149980545044, "prefillTokensPerSecond": 96.57516208327914, "promptTokens": 54, "residentMemoryAfterBytes": 15817162752, "residentMemoryBeforeBytes": 15817129984, "ttftSeconds": 0.559548020362854 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.8668528774133106 }, "endToEndSpeedups": { "adaptive": 2.7216101728632554 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 20.13426899909973, "decodeTokensPerSecond": 12.714640894658087, "elapsedSeconds": 20.875707983970642, "endToEndTokensPerSecond": 12.263057147406398, "expectedMTPActive": false, "firstTokenSeconds": 0.05099892616271973, "generatedTokens": 256, "loadSeconds": 2.193450927734375e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.7390890121459961, "prefillTokensPerSecond": 73.06291815001723, "promptTokens": 54, "residentMemoryAfterBytes": 15397109760, "residentMemoryBeforeBytes": 15396847616, "ttftSeconds": 0.7901098728179932 }, { "acceleration": { "acceptanceRate": 0.7710843373493976, "acceptedDraftTokens": 192, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 249, "rounds": 64, "route": "mtp-speculative" }, "decodeSeconds": 7.023126006126404, "decodeTokensPerSecond": 36.45100483412749, "elapsedSeconds": 7.670351982116699, "endToEndTokensPerSecond": 33.375261082784704, "expectedMTPActive": true, "firstTokenSeconds": 0.00035691261291503906, "generatedTokens": 256, "loadSeconds": 1.895427703857422e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.6456530094146729, "prefillTokensPerSecond": 83.63625540745883, "promptTokens": 54, "residentMemoryAfterBytes": 15817195520, "residentMemoryBeforeBytes": 15817162752, "ttftSeconds": 0.6460288763046265 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-c30/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "16e51686bc2dd0d8ba236e991e720836494ac98f3dd354458999011a964d608f", "receipt": { "after": { "gpuDeviceUtilizationPercent": 77, "loadAverage": [ 4.4814453125, 8.5244140625, 9.13525390625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 50, "loadAverage": [ 9.93896484375, 10.96044921875, 9.97705078125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 40, "elapsedSeconds": 0.08319902420043945, "loadAverage": [ 9.93896484375, 10.96044921875, 9.97705078125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 2.1835098266601562, "loadAverage": [ 9.93896484375, 10.96044921875, 9.97705078125 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 4.286635160446167, "loadAverage": [ 9.46337890625, 10.8447265625, 9.94189453125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 6.381968021392822, "loadAverage": [ 9.46337890625, 10.8447265625, 9.94189453125 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 8.473695993423462, "loadAverage": [ 9.46337890625, 10.8447265625, 9.94189453125 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 10.568106174468994, "loadAverage": [ 9.18603515625, 10.76416015625, 9.91845703125 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 12.660989999771118, "loadAverage": [ 9.18603515625, 10.76416015625, 9.91845703125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 14.756610870361328, "loadAverage": [ 8.6103515625, 10.61865234375, 9.87158203125 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 16.854438066482544, "loadAverage": [ 8.6103515625, 10.61865234375, 9.87158203125 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 18.95078682899475, "loadAverage": [ 8.24072265625, 10.50830078125, 9.8369140625 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 21.046904802322388, "loadAverage": [ 8.24072265625, 10.50830078125, 9.8369140625 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 23.140676021575928, "loadAverage": [ 8.24072265625, 10.50830078125, 9.8369140625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 25.230878114700317, "loadAverage": [ 7.90087890625, 10.39990234375, 9.80224609375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 27.32303810119629, "loadAverage": [ 7.90087890625, 10.39990234375, 9.80224609375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 29.41638684272766, "loadAverage": [ 7.90869140625, 10.35986328125, 9.79150390625 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 31.509045124053955, "loadAverage": [ 7.90869140625, 10.35986328125, 9.79150390625 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 33.604341983795166, "loadAverage": [ 7.35498046875, 10.2041015625, 9.73974609375 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 35.70349907875061, "loadAverage": [ 7.35498046875, 10.2041015625, 9.73974609375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 37.79845094680786, "loadAverage": [ 7.35498046875, 10.2041015625, 9.73974609375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 39.90022897720337, "loadAverage": [ 7.24609375, 10.13427734375, 9.7177734375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 41.99768090248108, "loadAverage": [ 7.24609375, 10.13427734375, 9.7177734375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 44.10057878494263, "loadAverage": [ 6.90576171875, 10.015625, 9.67822265625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 46.201536893844604, "loadAverage": [ 6.90576171875, 10.015625, 9.67822265625 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 48.302613735198975, "loadAverage": [ 6.90576171875, 10.015625, 9.67822265625 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 50.40633296966553, "loadAverage": [ 6.3525390625, 9.84912109375, 9.62109375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 52.520609855651855, "loadAverage": [ 6.3525390625, 9.84912109375, 9.62109375 ] }, { "deviceUtilizationPercent": 35, "elapsedSeconds": 54.633506774902344, "loadAverage": [ 6.1640625, 9.751953125, 9.587890625 ] }, { "deviceUtilizationPercent": 39, "elapsedSeconds": 56.75975203514099, "loadAverage": [ 6.1640625, 9.751953125, 9.587890625 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 58.87088584899902, "loadAverage": [ 5.99072265625, 9.65625, 9.5546875 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 60.99455499649048, "loadAverage": [ 5.99072265625, 9.65625, 9.5546875 ] }, { "deviceUtilizationPercent": 33, "elapsedSeconds": 63.10622191429138, "loadAverage": [ 5.99072265625, 9.65625, 9.5546875 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 65.21710705757141, "loadAverage": [ 5.9111328125, 9.57861328125, 9.52783203125 ] }, { "deviceUtilizationPercent": 33, "elapsedSeconds": 67.33113598823547, "loadAverage": [ 5.9111328125, 9.57861328125, 9.52783203125 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 69.4352080821991, "loadAverage": [ 5.677734375, 9.46923828125, 9.4892578125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 71.54713201522827, "loadAverage": [ 5.677734375, 9.46923828125, 9.4892578125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 73.66341996192932, "loadAverage": [ 5.3828125, 9.34521484375, 9.4453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 75.7834677696228, "loadAverage": [ 5.3828125, 9.34521484375, 9.4453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 77.88332891464233, "loadAverage": [ 5.3828125, 9.34521484375, 9.4453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 80.00029301643372, "loadAverage": [ 5.27197265625, 9.25634765625, 9.4130859375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 82.10288381576538, "loadAverage": [ 5.27197265625, 9.25634765625, 9.4130859375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 84.21901392936707, "loadAverage": [ 5.009765625, 9.1357421875, 9.36962890625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 86.32019281387329, "loadAverage": [ 5.009765625, 9.1357421875, 9.36962890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 88.42929196357727, "loadAverage": [ 5.009765625, 9.1357421875, 9.36962890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 90.53942108154297, "loadAverage": [ 4.8486328125, 9.03369140625, 9.33203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 92.66448593139648, "loadAverage": [ 4.8486328125, 9.03369140625, 9.33203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 94.76693987846375, "loadAverage": [ 4.6201171875, 8.91650390625, 9.2890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 96.86728096008301, "loadAverage": [ 4.6201171875, 8.91650390625, 9.2890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 98.98701500892639, "loadAverage": [ 4.330078125, 8.78466796875, 9.240234375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 101.10234713554382, "loadAverage": [ 4.330078125, 8.78466796875, 9.240234375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 103.19229507446289, "loadAverage": [ 4.330078125, 8.78466796875, 9.240234375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 105.2909369468689, "loadAverage": [ 4.30322265625, 8.705078125, 9.20947265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 107.40547204017639, "loadAverage": [ 4.30322265625, 8.705078125, 9.20947265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 109.52663397789001, "loadAverage": [ 4.19873046875, 8.6103515625, 9.1728515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 111.6428530216217, "loadAverage": [ 4.19873046875, 8.6103515625, 9.1728515625 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 113.82783699035645, "loadAverage": [ 4.2626953125, 8.55029296875, 9.14794921875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 116.36687278747559, "loadAverage": [ 4.2626953125, 8.55029296875, 9.14794921875 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 118.87349891662598, "loadAverage": [ 4.4814453125, 8.5244140625, 9.13525390625 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15815147520, "processSeconds": 121.09185099601746, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "1", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.3", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "1", "MERERUN_Q35_SCOPED_COMPILE": "1" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788622503.062458, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.4522703439435756 }, "endToEndSpeedups": { "adaptive": 1.4490255085590151 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.49407114624505927, "acceptedDraftTokens": 125, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 11.374314069747925, "decodeTokensPerSecond": 22.50685170377693, "elapsedSeconds": 12.068704009056091, "endToEndTokensPerSecond": 21.21188818682629, "expectedMTPActive": true, "firstTokenSeconds": 0.0003960132598876953, "generatedTokens": 256, "loadSeconds": 1.990795135498047e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.6927779912948608, "prefillTokensPerSecond": 86.60783216836163, "promptTokens": 60, "residentMemoryAfterBytes": 15814787072, "residentMemoryBeforeBytes": 15814557696, "ttftSeconds": 0.6931939125061035 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.51857900619507, "decodeTokensPerSecond": 15.497701097896536, "elapsedSeconds": 17.487859964370728, "endToEndTokensPerSecond": 14.63872655210913, "expectedMTPActive": false, "firstTokenSeconds": 0.05291104316711426, "generatedTokens": 256, "loadSeconds": 2.8967857360839844e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.9662530422210693, "prefillTokensPerSecond": 62.09553541180218, "promptTokens": 60, "residentMemoryAfterBytes": 15426945024, "residentMemoryBeforeBytes": 15426895872, "ttftSeconds": 1.0191930532455444 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.3804458779822655 }, "endToEndSpeedups": { "adaptive": 1.3551175307244239 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.49407114624505927, "acceptedDraftTokens": 125, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 11.735543012619019, "decodeTokensPerSecond": 21.814073684083283, "elapsedSeconds": 12.62632405757904, "endToEndTokensPerSecond": 20.275101354327603, "expectedMTPActive": true, "firstTokenSeconds": 0.00040400028228759766, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.8891839981079102, "prefillTokensPerSecond": 67.47759758123591, "promptTokens": 60, "residentMemoryAfterBytes": 15814950912, "residentMemoryBeforeBytes": 15814885376, "ttftSeconds": 0.8896059989929199 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.200281977653503, "decodeTokensPerSecond": 15.802194082369905, "elapsedSeconds": 17.110153079032898, "endToEndTokensPerSecond": 14.96187665987087, "expectedMTPActive": false, "firstTokenSeconds": 0.049934983253479004, "generatedTokens": 256, "loadSeconds": 5.0067901611328125e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.9059699773788452, "prefillTokensPerSecond": 66.22736017543558, "promptTokens": 60, "residentMemoryAfterBytes": 15426945024, "residentMemoryBeforeBytes": 15426977792, "ttftSeconds": 0.9559550285339355 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.4536891035427015 }, "endToEndSpeedups": { "adaptive": 1.4526044060963965 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.49407114624505927, "acceptedDraftTokens": 125, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 11.530770897865295, "decodeTokensPerSecond": 22.201464435252422, "elapsedSeconds": 12.270874977111816, "endToEndTokensPerSecond": 20.862407976407763, "expectedMTPActive": true, "firstTokenSeconds": 0.00038993358612060547, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.7384730577468872, "prefillTokensPerSecond": 81.2487325984005, "promptTokens": 60, "residentMemoryAfterBytes": 15815131136, "residentMemoryBeforeBytes": 15814950912, "ttftSeconds": 0.7388830184936523 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.762156009674072, "decodeTokensPerSecond": 15.272498349988673, "elapsedSeconds": 17.824727058410645, "endToEndTokensPerSecond": 14.362071248614475, "expectedMTPActive": false, "firstTokenSeconds": 0.059265971183776855, "generatedTokens": 256, "loadSeconds": 6.604194641113281e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 1.0589549541473389, "prefillTokensPerSecond": 56.65963388245486, "promptTokens": 60, "residentMemoryAfterBytes": 15427043328, "residentMemoryBeforeBytes": 15426945024, "ttftSeconds": 1.1182869672775269 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-current/qwen-code", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "0a93f096b27d690d0e3974c1bc302fd92ee8edb9594f7e5ff71baa28ce497fa8", "receipt": { "after": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 9.98681640625, 8.39208984375, 8.03271484375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 6.60400390625, 6.55078125, 7.37744140625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08145713806152344, "loadAverage": [ 6.60400390625, 6.55078125, 7.37744140625 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 2.1803479194641113, "loadAverage": [ 6.60400390625, 6.55078125, 7.37744140625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 4.291781187057495, "loadAverage": [ 6.39501953125, 6.50830078125, 7.357421875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 6.412467002868652, "loadAverage": [ 6.39501953125, 6.50830078125, 7.357421875 ] }, { "deviceUtilizationPercent": 33, "elapsedSeconds": 8.56551718711853, "loadAverage": [ 6.39501953125, 6.50830078125, 7.357421875 ] }, { "deviceUtilizationPercent": 2, "elapsedSeconds": 10.715137004852295, "loadAverage": [ 6.5234375, 6.53271484375, 7.36083984375 ] }, { "deviceUtilizationPercent": 4, "elapsedSeconds": 12.898132085800171, "loadAverage": [ 6.5234375, 6.53271484375, 7.36083984375 ] }, { "deviceUtilizationPercent": 35, "elapsedSeconds": 15.06649899482727, "loadAverage": [ 8.5634765625, 6.955078125, 7.5048828125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 17.18791389465332, "loadAverage": [ 8.5634765625, 6.955078125, 7.5048828125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 19.311448097229004, "loadAverage": [ 8.35791015625, 6.93896484375, 7.49560546875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 21.422831058502197, "loadAverage": [ 8.35791015625, 6.93896484375, 7.49560546875 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 23.53557300567627, "loadAverage": [ 8.35791015625, 6.93896484375, 7.49560546875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 25.675463914871216, "loadAverage": [ 8.2490234375, 6.93994140625, 7.49267578125 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 27.800579071044922, "loadAverage": [ 8.2490234375, 6.93994140625, 7.49267578125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 29.962523937225342, "loadAverage": [ 8.06884765625, 6.92431640625, 7.48388671875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 32.118709087371826, "loadAverage": [ 8.06884765625, 6.92431640625, 7.48388671875 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 34.25584411621094, "loadAverage": [ 8.22314453125, 6.97509765625, 7.49853515625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 36.388510942459106, "loadAverage": [ 8.22314453125, 6.97509765625, 7.49853515625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 39.00379300117493, "loadAverage": [ 10.927734375, 7.55615234375, 7.70068359375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 41.446017026901245, "loadAverage": [ 10.927734375, 7.55615234375, 7.70068359375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 43.961341857910156, "loadAverage": [ 13.33544921875, 8.111328125, 7.8955078125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 46.43530297279358, "loadAverage": [ 13.33544921875, 8.111328125, 7.8955078125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 48.917296171188354, "loadAverage": [ 16.27099609375, 8.806640625, 8.14208984375 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 51.45944023132324, "loadAverage": [ 16.27099609375, 8.806640625, 8.14208984375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 53.66731905937195, "loadAverage": [ 16.27099609375, 8.806640625, 8.14208984375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 55.78848195075989, "loadAverage": [ 15.5283203125, 8.7763671875, 8.13525390625 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 57.935641050338745, "loadAverage": [ 15.5283203125, 8.7763671875, 8.13525390625 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 60.06643199920654, "loadAverage": [ 14.76513671875, 8.72998046875, 8.12255859375 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 62.18958806991577, "loadAverage": [ 14.76513671875, 8.72998046875, 8.12255859375 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 64.33413600921631, "loadAverage": [ 14.30322265625, 8.734375, 8.12744140625 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 66.48514008522034, "loadAverage": [ 14.30322265625, 8.734375, 8.12744140625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 68.64633393287659, "loadAverage": [ 14.30322265625, 8.734375, 8.12744140625 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 70.75287818908691, "loadAverage": [ 14.03857421875, 8.77197265625, 8.14404296875 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 72.91491293907166, "loadAverage": [ 14.03857421875, 8.77197265625, 8.14404296875 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 75.06115889549255, "loadAverage": [ 13.234375, 8.6923828125, 8.11962890625 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 77.17303919792175, "loadAverage": [ 13.234375, 8.6923828125, 8.11962890625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 79.2857551574707, "loadAverage": [ 12.57470703125, 8.630859375, 8.10107421875 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 81.42498898506165, "loadAverage": [ 12.57470703125, 8.630859375, 8.10107421875 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 83.61014914512634, "loadAverage": [ 12.57470703125, 8.630859375, 8.10107421875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 85.77714610099792, "loadAverage": [ 12.2880859375, 8.63671875, 8.10595703125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 87.91169619560242, "loadAverage": [ 12.2880859375, 8.63671875, 8.10595703125 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 90.05510807037354, "loadAverage": [ 11.5439453125, 8.54296875, 8.07568359375 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 92.21657609939575, "loadAverage": [ 11.5439453125, 8.54296875, 8.07568359375 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 94.34531712532043, "loadAverage": [ 11.01953125, 8.48388671875, 8.0576171875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 96.44545888900757, "loadAverage": [ 11.01953125, 8.48388671875, 8.0576171875 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 98.54156112670898, "loadAverage": [ 11.01953125, 8.48388671875, 8.0576171875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 100.63730502128601, "loadAverage": [ 10.376953125, 8.392578125, 8.02783203125 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 102.76449799537659, "loadAverage": [ 10.376953125, 8.392578125, 8.02783203125 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 104.91477704048157, "loadAverage": [ 10.0263671875, 8.3525390625, 8.015625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 107.02461910247803, "loadAverage": [ 10.0263671875, 8.3525390625, 8.015625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 109.15956997871399, "loadAverage": [ 9.46337890625, 8.263671875, 7.98583984375 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 111.29512190818787, "loadAverage": [ 9.46337890625, 8.263671875, 7.98583984375 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 113.46925806999207, "loadAverage": [ 9.46337890625, 8.263671875, 7.98583984375 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 115.66775798797607, "loadAverage": [ 9.98681640625, 8.39208984375, 8.03271484375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15839952896, "processSeconds": 117.77565002441406, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "0", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "0", "MERERUN_Q35_SCOPED_COMPILE": "0" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788621902.651583, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.5776087322452335 }, "endToEndSpeedups": { "adaptive": 2.49221627764203 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 22.356317043304443, "decodeTokensPerSecond": 11.450902199325812, "elapsedSeconds": 23.05906903743744, "endToEndTokensPerSecond": 11.101922613804245, "expectedMTPActive": false, "firstTokenSeconds": 0.08319807052612305, "generatedTokens": 256, "loadSeconds": 5.4001808166503906e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.6990020275115967, "prefillTokensPerSecond": 77.25299480494586, "promptTokens": 54, "residentMemoryAfterBytes": 15399993344, "residentMemoryBeforeBytes": 15396880384, "ttftSeconds": 0.7822540998458862 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 8.673277974128723, "decodeTokensPerSecond": 29.515945501068362, "elapsedSeconds": 9.252434968948364, "endToEndTokensPerSecond": 27.668392251245088, "expectedMTPActive": true, "firstTokenSeconds": 0.0003470182418823242, "generatedTokens": 256, "loadSeconds": 6.592273712158203e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.5769939422607422, "prefillTokensPerSecond": 93.58850421968127, "promptTokens": 54, "residentMemoryAfterBytes": 15839313920, "residentMemoryBeforeBytes": 15838871552, "ttftSeconds": 0.5774068832397461 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.6794321428141084 }, "endToEndSpeedups": { "adaptive": 1.6228639454897986 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 14.338712930679321, "decodeTokensPerSecond": 17.853764228186662, "elapsedSeconds": 14.703480005264282, "endToEndTokensPerSecond": 17.410844229280716, "expectedMTPActive": false, "firstTokenSeconds": 0.04106104373931885, "generatedTokens": 256, "loadSeconds": 2.7894973754882812e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.36246800422668457, "prefillTokensPerSecond": 148.97866672454995, "promptTokens": 54, "residentMemoryAfterBytes": 15400173568, "residentMemoryBeforeBytes": 15400108032, "ttftSeconds": 0.4035569429397583 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 8.53783404827118, "decodeTokensPerSecond": 29.984185515041403, "elapsedSeconds": 9.060204982757568, "endToEndTokensPerSecond": 28.255431360238795, "expectedMTPActive": true, "firstTokenSeconds": 0.0002770423889160156, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.5208230018615723, "prefillTokensPerSecond": 103.68205668142221, "promptTokens": 54, "residentMemoryAfterBytes": 15839870976, "residentMemoryBeforeBytes": 15839313920, "ttftSeconds": 0.5211180448532104 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 3.0422307982424552 }, "endToEndSpeedups": { "adaptive": 2.9501458836273957 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 24.706866025924683, "decodeTokensPerSecond": 10.361492215620613, "elapsedSeconds": 25.486027002334595, "endToEndTokensPerSecond": 10.044719797893553, "expectedMTPActive": false, "firstTokenSeconds": 0.08480799198150635, "generatedTokens": 256, "loadSeconds": 4.303455352783203e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.7702270746231079, "prefillTokensPerSecond": 70.1091947805439, "promptTokens": 54, "residentMemoryAfterBytes": 15400763392, "residentMemoryBeforeBytes": 15400173568, "ttftSeconds": 0.8550781011581421 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 8.121299028396606, "decodeTokensPerSecond": 31.522050734110483, "elapsedSeconds": 8.638903975486755, "endToEndTokensPerSecond": 29.63338876394627, "expectedMTPActive": true, "firstTokenSeconds": 0.0004470348358154297, "generatedTokens": 256, "loadSeconds": 2.205371856689453e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.5154309272766113, "prefillTokensPerSecond": 104.76670518263322, "promptTokens": 54, "residentMemoryAfterBytes": 15839838208, "residentMemoryBeforeBytes": 15839870976, "ttftSeconds": 0.5159000158309937 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "finish-selection/qwen-current/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "6c0d268019b0e400c732c139a61c38fc8f0e94666531e3b1d2c097dd2aefed2a", "receipt": { "after": { "gpuDeviceUtilizationPercent": 13, "loadAverage": [ 12.73046875, 10.03955078125, 8.75439453125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 9.98681640625, 8.39208984375, 8.03271484375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7276.25M free = 915.75M (encrypted)", "swapUsedMiB": 7276.25 }, "binarySHA256": "cdec91928873110a516e18abcf0db7904cde7398977a7e5b9317c0776a3c1737", "buildSourceRevision": "f847be2ef7ca2f8896829c3cc0b6a1a55ea72ea9", "collectorSHA256": "c5d800333764ac360dd08c27bd99ccf3bc3be6afb203f82ac9fddef41d7ce682", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3", "--mtp-block-size", "8" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 0, "elapsedSeconds": 0.08612608909606934, "loadAverage": [ 9.98681640625, 8.39208984375, 8.03271484375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 2.236851215362549, "loadAverage": [ 10.14794921875, 8.45166015625, 8.0556640625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 4.485138177871704, "loadAverage": [ 10.14794921875, 8.45166015625, 8.0556640625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 6.675924062728882, "loadAverage": [ 15.9013671875, 9.67236328125, 8.48876953125 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 8.870368003845215, "loadAverage": [ 15.9013671875, 9.67236328125, 8.48876953125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 11.029417037963867, "loadAverage": [ 15.4287109375, 9.677734375, 8.49755859375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 13.128684997558594, "loadAverage": [ 15.4287109375, 9.677734375, 8.49755859375 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 15.224062204360962, "loadAverage": [ 15.4287109375, 9.677734375, 8.49755859375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 17.330445051193237, "loadAverage": [ 14.35302734375, 9.5498046875, 8.45947265625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 19.428237915039062, "loadAverage": [ 14.35302734375, 9.5498046875, 8.45947265625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 21.526108026504517, "loadAverage": [ 15.12548828125, 9.78955078125, 8.55029296875 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 23.6200909614563, "loadAverage": [ 15.12548828125, 9.78955078125, 8.55029296875 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 25.713225841522217, "loadAverage": [ 15.12548828125, 9.78955078125, 8.55029296875 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 27.80753803253174, "loadAverage": [ 14.07421875, 9.66015625, 8.51171875 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 29.90483808517456, "loadAverage": [ 14.07421875, 9.66015625, 8.51171875 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 32.00054311752319, "loadAverage": [ 13.42724609375, 9.59912109375, 8.49658203125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 34.10477614402771, "loadAverage": [ 13.42724609375, 9.59912109375, 8.49658203125 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 36.20142722129822, "loadAverage": [ 12.751953125, 9.5224609375, 8.47607421875 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 38.29452705383301, "loadAverage": [ 12.751953125, 9.5224609375, 8.47607421875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 40.388498067855835, "loadAverage": [ 12.751953125, 9.5224609375, 8.47607421875 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 42.48245120048523, "loadAverage": [ 11.970703125, 9.4140625, 8.44384765625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 44.57988119125366, "loadAverage": [ 11.970703125, 9.4140625, 8.44384765625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 46.673940896987915, "loadAverage": [ 11.091796875, 9.27392578125, 8.39990234375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 48.76945900917053, "loadAverage": [ 11.091796875, 9.27392578125, 8.39990234375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 50.8612539768219, "loadAverage": [ 10.5234375, 9.18603515625, 8.3740234375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 52.955260038375854, "loadAverage": [ 10.5234375, 9.18603515625, 8.3740234375 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 55.06524610519409, "loadAverage": [ 10.5234375, 9.18603515625, 8.3740234375 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 57.16499900817871, "loadAverage": [ 9.9208984375, 9.0830078125, 8.34228515625 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 59.29031205177307, "loadAverage": [ 9.9208984375, 9.0830078125, 8.34228515625 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 61.44402599334717, "loadAverage": [ 9.6865234375, 9.04833984375, 8.333984375 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 63.557234048843384, "loadAverage": [ 9.6865234375, 9.04833984375, 8.333984375 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 65.66324281692505, "loadAverage": [ 9.6865234375, 9.04833984375, 8.333984375 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 67.79022097587585, "loadAverage": [ 9.23095703125, 8.96435546875, 8.30810546875 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 69.89347195625305, "loadAverage": [ 9.23095703125, 8.96435546875, 8.30810546875 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 71.99014687538147, "loadAverage": [ 8.65185546875, 8.8486328125, 8.27099609375 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 74.10067009925842, "loadAverage": [ 8.65185546875, 8.8486328125, 8.27099609375 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 76.19799590110779, "loadAverage": [ 8.119140625, 8.73486328125, 8.23388671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 78.29105496406555, "loadAverage": [ 8.119140625, 8.73486328125, 8.23388671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 80.39119911193848, "loadAverage": [ 8.119140625, 8.73486328125, 8.23388671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 82.48514914512634, "loadAverage": [ 7.62890625, 8.623046875, 8.197265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 84.58470320701599, "loadAverage": [ 7.62890625, 8.623046875, 8.197265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 86.69453001022339, "loadAverage": [ 7.498046875, 8.5791015625, 8.18408203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 88.79426288604736, "loadAverage": [ 7.498046875, 8.5791015625, 8.18408203125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 90.89651203155518, "loadAverage": [ 7.1376953125, 8.486328125, 8.1533203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 93.04852294921875, "loadAverage": [ 7.1376953125, 8.486328125, 8.1533203125 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 95.91297912597656, "loadAverage": [ 10.56982421875, 9.17529296875, 8.3984375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 98.89063215255737, "loadAverage": [ 10.56982421875, 9.17529296875, 8.3984375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 101.30351614952087, "loadAverage": [ 14.60791015625, 10.03564453125, 8.70654296875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 103.61294412612915, "loadAverage": [ 14.60791015625, 10.03564453125, 8.70654296875 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 105.97643899917603, "loadAverage": [ 15.43994140625, 10.28369140625, 8.8017578125 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 108.1181390285492, "loadAverage": [ 15.43994140625, 10.28369140625, 8.8017578125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 110.22887396812439, "loadAverage": [ 15.43994140625, 10.28369140625, 8.8017578125 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 112.3315041065216, "loadAverage": [ 14.443359375, 10.16259765625, 8.767578125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 114.4274468421936, "loadAverage": [ 14.443359375, 10.16259765625, 8.767578125 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 116.54293489456177, "loadAverage": [ 13.76708984375, 10.09326171875, 8.7509765625 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 118.64443683624268, "loadAverage": [ 13.76708984375, 10.09326171875, 8.7509765625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 120.74023604393005, "loadAverage": [ 13.76708984375, 10.09326171875, 8.7509765625 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 122.8449981212616, "loadAverage": [ 13.46533203125, 10.09130859375, 8.7578125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 124.93718218803406, "loadAverage": [ 13.46533203125, 10.09130859375, 8.7578125 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 127.026602268219, "loadAverage": [ 12.70703125, 9.98974609375, 8.7294921875 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 129.26920795440674, "loadAverage": [ 12.70703125, 9.98974609375, 8.7294921875 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15806726144, "processSeconds": 131.38125705718994, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": { "MERERUN_Q35_ASYNC_DECODE_BLOCKS": "0", "MERERUN_Q35_MTP_HEAD_COST_RATIO": "0.18", "MERERUN_Q35_MTP_PIPELINED_FALLBACK": "0", "MERERUN_Q35_SCOPED_COMPILE": "0" }, "sourceHead": "9319a217579cff3e3bbea002fb93451fe719a868", "startedUnixSeconds": 1788622020.551878, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.027829201567322 }, "endToEndSpeedups": { "adaptive": 1.0105509252242135 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 13.456549048423767, "decodeTokensPerSecond": 19.02419402469213, "elapsedSeconds": 14.077518105506897, "endToEndTokensPerSecond": 18.185023672593037, "expectedMTPActive": true, "firstTokenSeconds": 0.0003839731216430664, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.6193029880523682, "prefillTokensPerSecond": 96.8831107834513, "promptTokens": 60, "residentMemoryAfterBytes": 15806087168, "residentMemoryBeforeBytes": 15803072512, "ttftSeconds": 0.6197049617767334 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 13.831034064292908, "decodeTokensPerSecond": 18.509100535071788, "elapsedSeconds": 14.226048946380615, "endToEndTokensPerSecond": 17.995158105028974, "expectedMTPActive": false, "firstTokenSeconds": 0.03724396228790283, "generatedTokens": 256, "loadSeconds": 2.5033950805664062e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.3913620710372925, "prefillTokensPerSecond": 153.31071772226662, "promptTokens": 60, "residentMemoryAfterBytes": 15430025216, "residentMemoryBeforeBytes": 15429943296, "ttftSeconds": 0.428631067276001 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.3797964162489222 }, "endToEndSpeedups": { "adaptive": 1.3431540599329819 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 11.17688000202179, "decodeTokensPerSecond": 22.90442412853069, "elapsedSeconds": 12.036491990089417, "endToEndTokensPerSecond": 21.268655369918807, "expectedMTPActive": true, "firstTokenSeconds": 0.0003679990768432617, "generatedTokens": 256, "loadSeconds": 2.110004425048828e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.8579670190811157, "prefillTokensPerSecond": 69.93275809629618, "promptTokens": 60, "residentMemoryAfterBytes": 15806513152, "residentMemoryBeforeBytes": 15806201856, "ttftSeconds": 0.8583561182022095 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 15.421818971633911, "decodeTokensPerSecond": 16.59985767378498, "elapsedSeconds": 16.166863083839417, "endToEndTokensPerSecond": 15.834859160519555, "expectedMTPActive": false, "firstTokenSeconds": 0.03908097743988037, "generatedTokens": 256, "loadSeconds": 5.1021575927734375e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.7414500713348389, "prefillTokensPerSecond": 80.92250890472165, "promptTokens": 60, "residentMemoryAfterBytes": 15430107136, "residentMemoryBeforeBytes": 15430041600, "ttftSeconds": 0.780582070350647 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.9375156545679537 }, "endToEndSpeedups": { "adaptive": 1.8784972694003337 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 11.35870099067688, "decodeTokensPerSecond": 22.53778845046828, "elapsedSeconds": 12.081281900405884, "endToEndTokensPerSecond": 21.18980436930285, "expectedMTPActive": true, "firstTokenSeconds": 0.0003879070281982422, "generatedTokens": 256, "loadSeconds": 2.205371856689453e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.7209110260009766, "prefillTokensPerSecond": 83.22802375881365, "promptTokens": 60, "residentMemoryAfterBytes": 15806709760, "residentMemoryBeforeBytes": 15806529536, "ttftSeconds": 0.7213209867477417 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 22.00766098499298, "decodeTokensPerSecond": 11.632312955682403, "elapsedSeconds": 22.694655060768127, "endToEndTokensPerSecond": 11.280189071590824, "expectedMTPActive": false, "firstTokenSeconds": 0.08577299118041992, "generatedTokens": 256, "loadSeconds": 6.794929504394531e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.6828999519348145, "prefillTokensPerSecond": 87.86060070733062, "promptTokens": 60, "residentMemoryAfterBytes": 15430434816, "residentMemoryBeforeBytes": 15430107136, "ttftSeconds": 0.7687408924102783 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/ornith-chat", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "d799f13d05f14e5baf396753bfd2f6a27420a1e1c8aa09d864a5a49f8e0e7212", "receipt": { "after": { "gpuDeviceUtilizationPercent": 59, "loadAverage": [ 3.95751953125, 5.8369140625, 6.72509765625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 44, "loadAverage": [ 4.10595703125, 6.13623046875, 6.8701171875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 44, "elapsedSeconds": 0.08710670471191406, "loadAverage": [ 4.10595703125, 6.13623046875, 6.8701171875 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 2.186091899871826, "loadAverage": [ 3.93701171875, 6.0673828125, 6.84130859375 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 4.302156686782837, "loadAverage": [ 3.93701171875, 6.0673828125, 6.84130859375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 6.434554815292358, "loadAverage": [ 3.93701171875, 6.0673828125, 6.84130859375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 8.602156639099121, "loadAverage": [ 3.94189453125, 6.03271484375, 6.82421875 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 10.78649377822876, "loadAverage": [ 3.94189453125, 6.03271484375, 6.82421875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 12.975947856903076, "loadAverage": [ 3.8662109375, 5.98193359375, 6.8017578125 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 15.163794755935669, "loadAverage": [ 3.8662109375, 5.98193359375, 6.8017578125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 17.369791984558105, "loadAverage": [ 3.95654296875, 5.96533203125, 6.791015625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 19.527888774871826, "loadAverage": [ 3.95654296875, 5.96533203125, 6.791015625 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 21.636314868927002, "loadAverage": [ 3.95654296875, 5.96533203125, 6.791015625 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 23.759453773498535, "loadAverage": [ 4.1201171875, 5.9658203125, 6.7861328125 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 25.92161273956299, "loadAverage": [ 4.1201171875, 5.9658203125, 6.7861328125 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 28.098074674606323, "loadAverage": [ 3.9501953125, 5.89990234375, 6.7578125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 30.285287857055664, "loadAverage": [ 3.9501953125, 5.89990234375, 6.7578125 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 32.47701382637024, "loadAverage": [ 3.9541015625, 5.8681640625, 6.7412109375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 34.65864586830139, "loadAverage": [ 3.9541015625, 5.8681640625, 6.7412109375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9464102912, "processSeconds": 36.816322803497314, "profiled": false, "prompt": "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624135.6288462, "uncontended": false, "workload": "chat" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9066142418009261 }, "endToEndSpeedups": { "adaptive": 0.9365809555342175 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.30120481927710846, "acceptedDraftTokens": 50, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 166, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 4.568674087524414, "decodeTokensPerSecond": 56.03376277135943, "elapsedSeconds": 4.85992693901062, "endToEndTokensPerSecond": 52.67568899957913, "expectedMTPActive": true, "firstTokenSeconds": 0.0005400180816650391, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "adaptive", "prefillSeconds": 0.2889150381088257, "prefillTokensPerSecond": 335.73884085418564, "promptTokens": 97, "residentMemoryAfterBytes": 9463889920, "residentMemoryBeforeBytes": 9462841344, "ttftSeconds": 0.28946805000305176 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 4.142024993896484, "decodeTokensPerSecond": 61.80551792353521, "elapsedSeconds": 4.551715016365051, "endToEndTokensPerSecond": 56.24253695136624, "expectedMTPActive": false, "firstTokenSeconds": 0.03979003429412842, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "baseline", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "disabled", "prefillSeconds": 0.4073280096054077, "prefillTokensPerSecond": 238.13731860464776, "promptTokens": 97, "residentMemoryAfterBytes": 9335062528, "residentMemoryBeforeBytes": 9334571008, "ttftSeconds": 0.44712603092193604 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.893343424563443 }, "endToEndSpeedups": { "adaptive": 0.8932543727167191 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.30120481927710846, "acceptedDraftTokens": 50, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 166, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 4.114128947257996, "decodeTokensPerSecond": 62.22459317193257, "elapsedSeconds": 4.3158780336380005, "endToEndTokensPerSecond": 59.31585601000149, "expectedMTPActive": true, "firstTokenSeconds": 0.0002579689025878906, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "adaptive", "prefillSeconds": 0.1984010934829712, "prefillTokensPerSecond": 488.9085956994765, "promptTokens": 97, "residentMemoryAfterBytes": 9464037376, "residentMemoryBeforeBytes": 9463988224, "ttftSeconds": 0.19867205619812012 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.6753300428390503, "decodeTokensPerSecond": 69.65360852388916, "elapsedSeconds": 3.8551769256591797, "endToEndTokensPerSecond": 66.40421566546591, "expectedMTPActive": false, "firstTokenSeconds": 0.012106060981750488, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "baseline", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "disabled", "prefillSeconds": 0.17681705951690674, "prefillTokensPerSecond": 548.5896002626666, "promptTokens": 97, "residentMemoryAfterBytes": 9335078912, "residentMemoryBeforeBytes": 9335062528, "ttftSeconds": 0.18893611431121826 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.8703461287593358 }, "endToEndSpeedups": { "adaptive": 0.875117628072822 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.30120481927710846, "acceptedDraftTokens": 50, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 166, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 4.576416015625, "decodeTokensPerSecond": 55.93897039210456, "elapsedSeconds": 4.740123987197876, "endToEndTokensPerSecond": 54.007026122397775, "expectedMTPActive": true, "firstTokenSeconds": 0.000514984130859375, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "adaptive", "prefillSeconds": 0.16047203540802002, "prefillTokensPerSecond": 604.4666894974285, "promptTokens": 97, "residentMemoryAfterBytes": 9464102912, "residentMemoryBeforeBytes": 9464037376, "ttftSeconds": 0.16100001335144043 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.983065962791443, "decodeTokensPerSecond": 64.27209651847897, "elapsedSeconds": 4.148166060447693, "endToEndTokensPerSecond": 61.71401922428608, "expectedMTPActive": false, "firstTokenSeconds": 0.017727971076965332, "generatedTokens": 256, "loadSeconds": 1.2040138244628906e-05, "name": "baseline", "outputSHA256": "d1f8455830507237affb86ec31863cc982d098dd55ce897ae205dcfa8f89000e", "policy": "disabled", "prefillSeconds": 0.16223406791687012, "prefillTokensPerSecond": 597.9015458683036, "promptTokens": 97, "residentMemoryAfterBytes": 9335144448, "residentMemoryBeforeBytes": 9335078912, "ttftSeconds": 0.17997407913208008 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/ornith-code", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "ef695a3833e370f758ea044912029211908a4781e30178cd3707329b9755a7aa", "receipt": { "after": { "gpuDeviceUtilizationPercent": 90, "loadAverage": [ 4.66650390625, 7.09130859375, 7.27392578125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 72, "loadAverage": [ 5.85205078125, 7.72509765625, 7.498046875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 44, "elapsedSeconds": 0.134735107421875, "loadAverage": [ 5.85205078125, 7.72509765625, 7.498046875 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 2.227550983428955, "loadAverage": [ 5.38330078125, 7.5966796875, 7.4541015625 ] }, { "deviceUtilizationPercent": 42, "elapsedSeconds": 4.322177886962891, "loadAverage": [ 5.38330078125, 7.5966796875, 7.4541015625 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 6.417167901992798, "loadAverage": [ 5.38330078125, 7.5966796875, 7.4541015625 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 8.509385824203491, "loadAverage": [ 5.2724609375, 7.53662109375, 7.43359375 ] }, { "deviceUtilizationPercent": 44, "elapsedSeconds": 10.600883960723877, "loadAverage": [ 5.2724609375, 7.53662109375, 7.43359375 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 12.69225788116455, "loadAverage": [ 5.01025390625, 7.4443359375, 7.4013671875 ] }, { "deviceUtilizationPercent": 44, "elapsedSeconds": 14.787984848022461, "loadAverage": [ 5.01025390625, 7.4443359375, 7.4013671875 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 16.883733987808228, "loadAverage": [ 5.08935546875, 7.419921875, 7.39306640625 ] }, { "deviceUtilizationPercent": 47, "elapsedSeconds": 18.976210832595825, "loadAverage": [ 5.08935546875, 7.419921875, 7.39306640625 ] }, { "deviceUtilizationPercent": 44, "elapsedSeconds": 21.069261074066162, "loadAverage": [ 5.08935546875, 7.419921875, 7.39306640625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 23.171536922454834, "loadAverage": [ 5.001953125, 7.36279296875, 7.373046875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 25.28164291381836, "loadAverage": [ 5.001953125, 7.36279296875, 7.373046875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 27.39118194580078, "loadAverage": [ 4.92138671875, 7.306640625, 7.35302734375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 29.50066089630127, "loadAverage": [ 4.92138671875, 7.306640625, 7.35302734375 ] }, { "deviceUtilizationPercent": 41, "elapsedSeconds": 31.60195302963257, "loadAverage": [ 4.92138671875, 7.306640625, 7.35302734375 ] }, { "deviceUtilizationPercent": 44, "elapsedSeconds": 33.70350003242493, "loadAverage": [ 4.92724609375, 7.26806640625, 7.3388671875 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 35.814661741256714, "loadAverage": [ 4.92724609375, 7.26806640625, 7.3388671875 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 37.91559076309204, "loadAverage": [ 4.8525390625, 7.21337890625, 7.31884765625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 40.0178427696228, "loadAverage": [ 4.8525390625, 7.21337890625, 7.31884765625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 42.12201189994812, "loadAverage": [ 4.4638671875, 7.09326171875, 7.27587890625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 44.22876596450806, "loadAverage": [ 4.4638671875, 7.09326171875, 7.27587890625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 46.3476459980011, "loadAverage": [ 4.4638671875, 7.09326171875, 7.27587890625 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9467723776, "processSeconds": 48.45229697227478, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788623955.434987, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.5032358009761664 }, "endToEndSpeedups": { "adaptive": 0.5097982709707708 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.9574549198150635, "decodeTokensPerSecond": 130.78206675849597, "elapsedSeconds": 2.0365079641342163, "endToEndTokensPerSecond": 125.70537631500679, "expectedMTPActive": false, "firstTokenSeconds": 0.007560014724731445, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07733690738677979, "prefillTokensPerSecond": 698.2435918976369, "promptTokens": 54, "residentMemoryAfterBytes": 7700643840, "residentMemoryBeforeBytes": 7700594688, "ttftSeconds": 0.0849078893661499 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 3.8897370100021362, "decodeTokensPerSecond": 65.81421811853018, "elapsedSeconds": 3.994732975959778, "endToEndTokensPerSecond": 64.08438349712054, "expectedMTPActive": true, "firstTokenSeconds": 0.00018906593322753906, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.10337197780609131, "prefillTokensPerSecond": 522.3852841559735, "promptTokens": 54, "residentMemoryAfterBytes": 9467576320, "residentMemoryBeforeBytes": 9467084800, "ttftSeconds": 0.10356700420379639 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.5145088326544027 }, "endToEndSpeedups": { "adaptive": 0.5127918045602642 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.9333399534225464, "decodeTokensPerSecond": 132.41333969579907, "elapsedSeconds": 2.012495994567871, "endToEndTokensPerSecond": 127.20522211770616, "expectedMTPActive": false, "firstTokenSeconds": 0.008253931999206543, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07761597633361816, "prefillTokensPerSecond": 695.7330507303138, "promptTokens": 54, "residentMemoryAfterBytes": 7700873216, "residentMemoryBeforeBytes": 7700758528, "ttftSeconds": 0.08587586879730225 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 3.757641911506653, "decodeTokensPerSecond": 68.12783283475646, "elapsedSeconds": 3.9245868921279907, "endToEndTokensPerSecond": 65.22979539922777, "expectedMTPActive": true, "firstTokenSeconds": 0.0002689361572265625, "generatedTokens": 256, "loadSeconds": 6.9141387939453125e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.1652590036392212, "prefillTokensPerSecond": 326.75980618815794, "promptTokens": 54, "residentMemoryAfterBytes": 9467641856, "residentMemoryBeforeBytes": 9467576320, "ttftSeconds": 0.1655348539352417 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.5086490251551099 }, "endToEndSpeedups": { "adaptive": 0.5048625286941151 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 1.976104974746704, "decodeTokensPerSecond": 129.5477736615758, "elapsedSeconds": 2.055469036102295, "endToEndTokensPerSecond": 124.5457827403923, "expectedMTPActive": false, "firstTokenSeconds": 0.007930994033813477, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.07777106761932373, "prefillTokensPerSecond": 694.3456178886588, "promptTokens": 54, "residentMemoryAfterBytes": 7700987904, "residentMemoryBeforeBytes": 7700873216, "ttftSeconds": 0.08571302890777588 }, { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 3.885006904602051, "decodeTokensPerSecond": 65.89434878397536, "elapsedSeconds": 4.071344017982483, "endToEndTokensPerSecond": 62.87849881250233, "expectedMTPActive": true, "firstTokenSeconds": 0.0004469156265258789, "generatedTokens": 256, "loadSeconds": 5.9604644775390625e-06, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.18459808826446533, "prefillTokensPerSecond": 292.527406473661, "promptTokens": 54, "residentMemoryAfterBytes": 9467674624, "residentMemoryBeforeBytes": 9467641856, "ttftSeconds": 0.18505096435546875 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/ornith-math", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "c1cd43252f5da34c6cdd838116b5fd87df15e316a094aac21972e21acce3a292", "receipt": { "after": { "gpuDeviceUtilizationPercent": 52, "loadAverage": [ 5.79248046875, 5.6796875, 6.28564453125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 59, "loadAverage": [ 5.52734375, 5.56689453125, 6.271484375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Derive the sum of the first n squares using induction. Explain the base case and inductive step, then check your formula for n=10. Show each algebraic step.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 65, "elapsedSeconds": 0.08813071250915527, "loadAverage": [ 5.52734375, 5.56689453125, 6.271484375 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 2.1917686462402344, "loadAverage": [ 5.48486328125, 5.55712890625, 6.263671875 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 4.293680906295776, "loadAverage": [ 5.48486328125, 5.55712890625, 6.263671875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 6.672083854675293, "loadAverage": [ 6.72705078125, 5.8134765625, 6.349609375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 8.901358842849731, "loadAverage": [ 6.72705078125, 5.8134765625, 6.349609375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 11.092183828353882, "loadAverage": [ 6.90869140625, 5.8662109375, 6.36474609375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 13.286022901535034, "loadAverage": [ 6.90869140625, 5.8662109375, 6.36474609375 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 15.469047784805298, "loadAverage": [ 6.59521484375, 5.818359375, 6.3447265625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 17.649840831756592, "loadAverage": [ 6.59521484375, 5.818359375, 6.3447265625 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 19.754568815231323, "loadAverage": [ 6.59521484375, 5.818359375, 6.3447265625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 21.876179933547974, "loadAverage": [ 6.38720703125, 5.7880859375, 6.33056640625 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 24.025762796401978, "loadAverage": [ 6.38720703125, 5.7880859375, 6.33056640625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 26.196739673614502, "loadAverage": [ 6.03564453125, 5.72509765625, 6.30517578125 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 28.384493827819824, "loadAverage": [ 6.03564453125, 5.72509765625, 6.30517578125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 30.5664119720459, "loadAverage": [ 5.79248046875, 5.6796875, 6.28564453125 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 32.74526882171631, "loadAverage": [ 5.79248046875, 5.6796875, 6.28564453125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9463414784, "processSeconds": 34.885260820388794, "profiled": false, "prompt": "Derive the sum of the first n squares using induction. Explain the base case and inductive step, then check your formula for n=10. Show each algebraic step.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624512.0353742, "uncontended": false, "workload": "math" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.7936179556854549 }, "endToEndSpeedups": { "adaptive": 0.8759878992545894 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5975103734439834, "acceptedDraftTokens": 144, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 111, "route": "mtp-speculative" }, "decodeSeconds": 4.880236029624939, "decodeTokensPerSecond": 52.45647924526191, "elapsedSeconds": 5.136402010917664, "endToEndTokensPerSecond": 49.840335599873214, "expectedMTPActive": true, "firstTokenSeconds": 0.0005780458450317383, "generatedTokens": 256, "loadSeconds": 1.2040138244628906e-05, "name": "adaptive", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "adaptive", "prefillSeconds": 0.2538108825683594, "prefillTokensPerSecond": 193.05712782745923, "promptTokens": 49, "residentMemoryAfterBytes": 9463283712, "residentMemoryBeforeBytes": 9462497280, "ttftSeconds": 0.25440096855163574 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.873042941093445, "decodeTokensPerSecond": 66.09789870486837, "elapsedSeconds": 4.499426007270813, "endToEndTokensPerSecond": 56.89614621649934, "expectedMTPActive": false, "firstTokenSeconds": 0.07702696323394775, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "baseline", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "disabled", "prefillSeconds": 0.624398946762085, "prefillTokensPerSecond": 78.47546869528993, "promptTokens": 49, "residentMemoryAfterBytes": 9350840320, "residentMemoryBeforeBytes": 9350529024, "ttftSeconds": 0.7014338970184326 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0945506845878075 }, "endToEndSpeedups": { "adaptive": 1.1068687813203837 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5975103734439834, "acceptedDraftTokens": 144, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 111, "route": "mtp-speculative" }, "decodeSeconds": 3.1709870100021362, "decodeTokensPerSecond": 80.73196111889072, "elapsedSeconds": 3.269270896911621, "endToEndTokensPerSecond": 78.30492121097559, "expectedMTPActive": true, "firstTokenSeconds": 0.0005669593811035156, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "adaptive", "prefillSeconds": 0.0955209732055664, "prefillTokensPerSecond": 512.9763480795918, "promptTokens": 49, "residentMemoryAfterBytes": 9463398400, "residentMemoryBeforeBytes": 9463382016, "ttftSeconds": 0.09610092639923096 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.4708060026168823, "decodeTokensPerSecond": 73.75808380156764, "elapsedSeconds": 3.618653893470764, "endToEndTokensPerSecond": 70.74453858709941, "expectedMTPActive": false, "firstTokenSeconds": 0.013299942016601562, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "disabled", "prefillSeconds": 0.1454859972000122, "prefillTokensPerSecond": 336.8021730134994, "promptTokens": 49, "residentMemoryAfterBytes": 9350889472, "residentMemoryBeforeBytes": 9350840320, "ttftSeconds": 0.15879690647125244 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.1660492631455546 }, "endToEndSpeedups": { "adaptive": 1.1596014396946597 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5975103734439834, "acceptedDraftTokens": 144, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 111, "route": "mtp-speculative" }, "decodeSeconds": 3.123867988586426, "decodeTokensPerSecond": 81.94968575347576, "elapsedSeconds": 3.2323830127716064, "endToEndTokensPerSecond": 79.19853525665351, "expectedMTPActive": true, "firstTokenSeconds": 0.0002549886703491211, "generatedTokens": 256, "loadSeconds": 1.4066696166992188e-05, "name": "adaptive", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "adaptive", "prefillSeconds": 0.10540890693664551, "prefillTokensPerSecond": 464.8563525039752, "promptTokens": 49, "residentMemoryAfterBytes": 9463414784, "residentMemoryBeforeBytes": 9463398400, "ttftSeconds": 0.10567796230316162 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.642583966255188, "decodeTokensPerSecond": 70.2797800604126, "elapsedSeconds": 3.7482759952545166, "endToEndTokensPerSecond": 68.298065650477, "expectedMTPActive": false, "firstTokenSeconds": 0.01730799674987793, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "baseline", "outputSHA256": "b754509c3e8b857df33b3149639a416f0c2b9800c69f82c20bd0c24dcbb0ab3d", "policy": "disabled", "prefillSeconds": 0.10316002368927002, "prefillTokensPerSecond": 474.99019724533696, "promptTokens": 49, "residentMemoryAfterBytes": 9350922240, "residentMemoryBeforeBytes": 9350889472, "ttftSeconds": 0.12048101425170898 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "76fe20e1067bd851ec3e7c84a2492d2caaac0c0feb9d572d6ce2eabcc65d6095", "receipt": { "after": { "gpuDeviceUtilizationPercent": 56, "loadAverage": [ 6.251953125, 5.73681640625, 6.47265625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 49, "loadAverage": [ 6.59130859375, 5.75830078125, 6.50830078125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 49, "elapsedSeconds": 0.15110278129577637, "loadAverage": [ 6.59130859375, 5.75830078125, 6.50830078125 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 2.25698184967041, "loadAverage": [ 6.59130859375, 5.75830078125, 6.50830078125 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 4.368875026702881, "loadAverage": [ 6.59130859375, 5.75830078125, 6.50830078125 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 6.513379096984863, "loadAverage": [ 6.3837890625, 5.72900390625, 6.4931640625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 8.674722909927368, "loadAverage": [ 6.3837890625, 5.72900390625, 6.4931640625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 10.857219696044922, "loadAverage": [ 6.03271484375, 5.6669921875, 6.466796875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 13.038350105285645, "loadAverage": [ 6.03271484375, 5.6669921875, 6.466796875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 15.301129817962646, "loadAverage": [ 6.27001953125, 5.72216796875, 6.4814453125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 17.438050985336304, "loadAverage": [ 6.27001953125, 5.72216796875, 6.4814453125 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 19.541571855545044, "loadAverage": [ 6.27001953125, 5.72216796875, 6.4814453125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 21.651201963424683, "loadAverage": [ 5.927734375, 5.66015625, 6.455078125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 23.82900595664978, "loadAverage": [ 5.927734375, 5.66015625, 6.455078125 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 25.974029064178467, "loadAverage": [ 6.01318359375, 5.68212890625, 6.4580078125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 28.15084481239319, "loadAverage": [ 6.01318359375, 5.68212890625, 6.4580078125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 30.312415838241577, "loadAverage": [ 6.251953125, 5.73681640625, 6.47265625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 32.49711203575134, "loadAverage": [ 6.251953125, 5.73681640625, 6.47265625 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9467789312, "processSeconds": 34.68603181838989, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624322.635404, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9274857928765436 }, "endToEndSpeedups": { "adaptive": 0.9237074993346861 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.8196980953216553, "decodeTokensPerSecond": 67.02100365302361, "elapsedSeconds": 3.9254270792007446, "endToEndTokensPerSecond": 65.21583380224811, "expectedMTPActive": false, "firstTokenSeconds": 0.008144021034240723, "generatedTokens": 256, "loadSeconds": 9.894371032714844e-06, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.10330605506896973, "prefillTokensPerSecond": 580.798482334288, "promptTokens": 60, "residentMemoryAfterBytes": 7684177920, "residentMemoryBeforeBytes": 7683833856, "ttftSeconds": 0.11145997047424316 }, { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 4.118335962295532, "decodeTokensPerSecond": 62.16102871250634, "elapsedSeconds": 4.249642968177795, "endToEndTokensPerSecond": 60.2403547585011, "expectedMTPActive": true, "firstTokenSeconds": 0.0002300739288330078, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.12905800342559814, "prefillTokensPerSecond": 464.90723866335003, "promptTokens": 60, "residentMemoryAfterBytes": 9467707392, "residentMemoryBeforeBytes": 9467117568, "ttftSeconds": 0.12929606437683105 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.985096498568095 }, "endToEndSpeedups": { "adaptive": 0.9866084036958582 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.7071239948272705, "decodeTokensPerSecond": 69.05622805096597, "elapsedSeconds": 3.8291209936141968, "endToEndTokensPerSecond": 66.85607491299693, "expectedMTPActive": false, "firstTokenSeconds": 0.014528036117553711, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.11905992031097412, "prefillTokensPerSecond": 503.94792675222055, "promptTokens": 60, "residentMemoryAfterBytes": 7684374528, "residentMemoryBeforeBytes": 7684276224, "ttftSeconds": 0.1335989236831665 }, { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 3.7632089853286743, "decodeTokensPerSecond": 68.02704845732644, "elapsedSeconds": 3.881095051765442, "endToEndTokensPerSecond": 65.96076534728262, "expectedMTPActive": true, "firstTokenSeconds": 0.00022101402282714844, "generatedTokens": 256, "loadSeconds": 1.0013580322265625e-05, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.11526799201965332, "prefillTokensPerSecond": 520.5261143940977, "promptTokens": 60, "residentMemoryAfterBytes": 9467772928, "residentMemoryBeforeBytes": 9467707392, "ttftSeconds": 0.11549901962280273 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.1059734078328523 }, "endToEndSpeedups": { "adaptive": 1.1010550176757645 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 4.395290970802307, "decodeTokensPerSecond": 58.24415304938738, "elapsedSeconds": 4.496928930282593, "endToEndTokensPerSecond": 56.92773979061142, "expectedMTPActive": false, "firstTokenSeconds": 0.016224980354309082, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.09855794906616211, "prefillTokensPerSecond": 608.778901838977, "promptTokens": 60, "residentMemoryAfterBytes": 7684472832, "residentMemoryBeforeBytes": 7684390912, "ttftSeconds": 0.11479592323303223 }, { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 3.974138021469116, "decodeTokensPerSecond": 64.41648443436917, "elapsedSeconds": 4.084200024604797, "endToEndTokensPerSecond": 62.680573541392974, "expectedMTPActive": true, "firstTokenSeconds": 0.00022304058074951172, "generatedTokens": 256, "loadSeconds": 1.1086463928222656e-05, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.10719001293182373, "prefillTokensPerSecond": 559.7536408374343, "promptTokens": 60, "residentMemoryAfterBytes": 9467805696, "residentMemoryBeforeBytes": 9467772928, "ttftSeconds": 0.10742413997650146 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/ornith-summary", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "f5535d654d17756b60f2d7f1f8bface5f5c5e853ed0f50bc89819863363d3afa", "receipt": { "after": { "gpuDeviceUtilizationPercent": 64, "loadAverage": [ 8.35107421875, 6.3984375, 6.40625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 47, "loadAverage": [ 6.49072265625, 5.58837890625, 6.125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Summarize this fictional incident report for a manager. Explain the timeline, user impact, cause, recovery, and three concrete follow-up actions. Distinguish confirmed facts from open questions. Use clear paragraphs and a short action list.\n\n09:02: The team deployed version 2.8 of the document search service. 09:07: Monitoring showed the 95th percentile search latency rising from 180 milliseconds to 4.2 seconds. 09:10: Support received six reports of slow searches; document uploads and downloads remained available. 09:13: The on-call engineer paused the rollout at 40 percent of traffic. 09:17: Logs showed repeated cache misses for queries containing a language filter. A new cache key included a randomly generated request identifier. 09:21: The engineer rolled traffic back to version 2.7. 09:26: Latency returned below 250 milliseconds. 09:32: The team confirmed no documents were lost or modified. Approximately 1,800 search requests experienced elevated latency; 73 timed out. The exact number of affected users is still being calculated. The release tests covered result correctness but did not assert cache reuse. A separate warning about database connection count appeared at 09:15; the team has not established whether it was a cause or an effect. Proposed follow-ups include a deterministic cache-key unit test, a staging workload with repeated filtered queries, and an automatic rollback threshold for latency.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 54, "elapsedSeconds": 0.16057491302490234, "loadAverage": [ 6.49072265625, 5.58837890625, 6.125 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 2.2684128284454346, "loadAverage": [ 9.333984375, 6.1923828125, 6.3349609375 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 4.801259994506836, "loadAverage": [ 9.333984375, 6.1923828125, 6.3349609375 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 7.117790937423706, "loadAverage": [ 10.50830078125, 6.48779296875, 6.43798828125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 9.274858951568604, "loadAverage": [ 10.50830078125, 6.48779296875, 6.43798828125 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 11.460591793060303, "loadAverage": [ 9.98681640625, 6.4462890625, 6.42333984375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 13.61159610748291, "loadAverage": [ 9.98681640625, 6.4462890625, 6.42333984375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 15.815939903259277, "loadAverage": [ 9.74755859375, 6.455078125, 6.42626953125 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 18.004486799240112, "loadAverage": [ 9.74755859375, 6.455078125, 6.42626953125 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 20.177622079849243, "loadAverage": [ 9.74755859375, 6.455078125, 6.42626953125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 22.31479287147522, "loadAverage": [ 9.767578125, 6.513671875, 6.44677734375 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 24.434377908706665, "loadAverage": [ 9.767578125, 6.513671875, 6.44677734375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 26.571160793304443, "loadAverage": [ 9.3056640625, 6.4716796875, 6.43212890625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 28.69957184791565, "loadAverage": [ 9.3056640625, 6.4716796875, 6.43212890625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 30.863296031951904, "loadAverage": [ 8.96044921875, 6.44677734375, 6.42333984375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 33.00080990791321, "loadAverage": [ 8.96044921875, 6.44677734375, 6.42333984375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 35.19631385803223, "loadAverage": [ 8.96044921875, 6.44677734375, 6.42333984375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 37.36798691749573, "loadAverage": [ 8.64306640625, 6.42236328125, 6.41455078125 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 39.561599016189575, "loadAverage": [ 8.64306640625, 6.42236328125, 6.41455078125 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9470541824, "processSeconds": 41.72237682342529, "profiled": false, "prompt": "Summarize this fictional incident report for a manager. Explain the timeline, user impact, cause, recovery, and three concrete follow-up actions. Distinguish confirmed facts from open questions. Use clear paragraphs and a short action list.\n\n09:02: The team deployed version 2.8 of the document search service. 09:07: Monitoring showed the 95th percentile search latency rising from 180 milliseconds to 4.2 seconds. 09:10: Support received six reports of slow searches; document uploads and downloads remained available. 09:13: The on-call engineer paused the rollout at 40 percent of traffic. 09:17: Logs showed repeated cache misses for queries containing a language filter. A new cache key included a randomly generated request identifier. 09:21: The engineer rolled traffic back to version 2.7. 09:26: Latency returned below 250 milliseconds. 09:32: The team confirmed no documents were lost or modified. Approximately 1,800 search requests experienced elevated latency; 73 timed out. The exact number of affected users is still being calculated. The release tests covered result correctness but did not assert cache reuse. A separate warning about database connection count appeared at 09:15; the team has not established whether it was a cause or an effect. Proposed follow-ups include a deterministic cache-key unit test, a staging workload with repeated filtered queries, and an automatic rollback threshold for latency.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624697.16877, "uncontended": false, "workload": "summary" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.2238893212688902 }, "endToEndSpeedups": { "adaptive": 0.9635954531881853 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.5036720037460327, "decodeTokensPerSecond": 73.06620018263457, "elapsedSeconds": 3.8215569257736206, "endToEndTokensPerSecond": 66.988404195543, "expectedMTPActive": false, "firstTokenSeconds": 0.010655999183654785, "generatedTokens": 256, "loadSeconds": 1.4066696166992188e-05, "name": "baseline", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "disabled", "prefillSeconds": 0.3154240846633911, "prefillTokensPerSecond": 1068.4028784917737, "promptTokens": 337, "residentMemoryAfterBytes": 7701495808, "residentMemoryBeforeBytes": 7701397504, "ttftSeconds": 0.3260941505432129 }, { "acceleration": { "acceptanceRate": 0.5375494071146245, "acceptedDraftTokens": 136, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 120, "route": "mtp-speculative" }, "decodeSeconds": 2.8627359867095947, "decodeTokensPerSecond": 89.4249421492215, "elapsedSeconds": 3.965934991836548, "endToEndTokensPerSecond": 64.54972169915759, "expectedMTPActive": true, "firstTokenSeconds": 0.00028896331787109375, "generatedTokens": 256, "loadSeconds": 1.0013580322265625e-05, "name": "adaptive", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "adaptive", "prefillSeconds": 1.1002739667892456, "prefillTokensPerSecond": 306.28735221593354, "promptTokens": 337, "residentMemoryAfterBytes": 9470197760, "residentMemoryBeforeBytes": 9470132224, "ttftSeconds": 1.100572943687439 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9305908973135506 }, "endToEndSpeedups": { "adaptive": 0.8083005551328226 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.052298069000244, "decodeTokensPerSecond": 83.87123217092974, "elapsedSeconds": 3.5733100175857544, "endToEndTokensPerSecond": 71.64225850545205, "expectedMTPActive": false, "firstTokenSeconds": 0.010917067527770996, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "baseline", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "disabled", "prefillSeconds": 0.5182070732116699, "prefillTokensPerSecond": 650.3191820817678, "promptTokens": 337, "residentMemoryAfterBytes": 7702331392, "residentMemoryBeforeBytes": 7701594112, "ttftSeconds": 0.529137134552002 }, { "acceleration": { "acceptanceRate": 0.5375494071146245, "acceptedDraftTokens": 136, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 120, "route": "mtp-speculative" }, "decodeSeconds": 3.2799569368362427, "decodeTokensPerSecond": 78.04980520473865, "elapsedSeconds": 4.420768976211548, "endToEndTokensPerSecond": 57.908477320926075, "expectedMTPActive": true, "firstTokenSeconds": 0.0002969503402709961, "generatedTokens": 256, "loadSeconds": 0.00012493133544921875, "name": "adaptive", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "adaptive", "prefillSeconds": 1.1369750499725342, "prefillTokensPerSecond": 296.4005234839066, "promptTokens": 337, "residentMemoryAfterBytes": 9470476288, "residentMemoryBeforeBytes": 9470197760, "ttftSeconds": 1.1373969316482544 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.3058649824450512 }, "endToEndSpeedups": { "adaptive": 1.179889300697985 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 4.09818696975708, "decodeTokensPerSecond": 62.46664729773771, "elapsedSeconds": 4.672605991363525, "endToEndTokensPerSecond": 54.7874142337638, "expectedMTPActive": false, "firstTokenSeconds": 0.021971940994262695, "generatedTokens": 256, "loadSeconds": 1.5020370483398438e-05, "name": "baseline", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "disabled", "prefillSeconds": 0.5709220170974731, "prefillTokensPerSecond": 590.2732595833035, "promptTokens": 337, "residentMemoryAfterBytes": 7702347776, "residentMemoryBeforeBytes": 7702331392, "ttftSeconds": 0.5929089784622192 }, { "acceleration": { "acceptanceRate": 0.5375494071146245, "acceptedDraftTokens": 136, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 253, "rounds": 120, "route": "mtp-speculative" }, "decodeSeconds": 3.1382930278778076, "decodeTokensPerSecond": 81.57300727686146, "elapsedSeconds": 3.9602071046829224, "endToEndTokensPerSecond": 64.6430838673264, "expectedMTPActive": true, "firstTokenSeconds": 0.0003420114517211914, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "a69b578276c6d560db5ba4730e6a6716506a19f3e2f0b8ee3af6cb09f3bb0aa7", "policy": "adaptive", "prefillSeconds": 0.8189549446105957, "prefillTokensPerSecond": 411.5000492001912, "promptTokens": 337, "residentMemoryAfterBytes": 9470541824, "residentMemoryBeforeBytes": 9470476288, "ttftSeconds": 0.8193099498748779 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/qwen-chat", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "c454db9ef4eef5183d8b9f20d9b5d196dd3e417e87b344072c86bf757bdc56ee", "receipt": { "after": { "gpuDeviceUtilizationPercent": 93, "loadAverage": [ 6.3818359375, 5.70361328125, 6.49365234375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 46, "loadAverage": [ 3.95751953125, 5.8369140625, 6.72509765625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 45, "elapsedSeconds": 0.1225290298461914, "loadAverage": [ 3.95751953125, 5.8369140625, 6.72509765625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 2.2237277030944824, "loadAverage": [ 3.95751953125, 5.8369140625, 6.72509765625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 4.3388848304748535, "loadAverage": [ 3.95751953125, 5.8369140625, 6.72509765625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 6.441427946090698, "loadAverage": [ 3.80029296875, 5.77294921875, 6.697265625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 8.532967805862427, "loadAverage": [ 3.80029296875, 5.77294921875, 6.697265625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 10.648180961608887, "loadAverage": [ 3.49560546875, 5.6767578125, 6.65771484375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 12.75586199760437, "loadAverage": [ 3.49560546875, 5.6767578125, 6.65771484375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 14.868896007537842, "loadAverage": [ 3.37548828125, 5.615234375, 6.63037109375 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 16.982999801635742, "loadAverage": [ 3.37548828125, 5.615234375, 6.63037109375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 19.097590923309326, "loadAverage": [ 3.37548828125, 5.615234375, 6.63037109375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 21.209198713302612, "loadAverage": [ 3.18505859375, 5.53857421875, 6.59716796875 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 23.33631992340088, "loadAverage": [ 3.18505859375, 5.53857421875, 6.59716796875 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 25.453585863113403, "loadAverage": [ 3.009765625, 5.462890625, 6.56396484375 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 27.572118997573853, "loadAverage": [ 3.009765625, 5.462890625, 6.56396484375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 29.673107862472534, "loadAverage": [ 3.0888671875, 5.4384765625, 6.548828125 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 31.780720710754395, "loadAverage": [ 3.0888671875, 5.4384765625, 6.548828125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 33.89406991004944, "loadAverage": [ 3.0888671875, 5.4384765625, 6.548828125 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 36.007766008377075, "loadAverage": [ 3.08154296875, 5.39794921875, 6.52783203125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 38.12344193458557, "loadAverage": [ 3.08154296875, 5.39794921875, 6.52783203125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 40.2421817779541, "loadAverage": [ 3.31494140625, 5.40771484375, 6.5244140625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 42.35401177406311, "loadAverage": [ 3.31494140625, 5.40771484375, 6.5244140625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 44.47113871574402, "loadAverage": [ 3.31494140625, 5.40771484375, 6.5244140625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 46.58198881149292, "loadAverage": [ 3.28955078125, 5.36767578125, 6.50341796875 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 48.680743932724, "loadAverage": [ 3.28955078125, 5.36767578125, 6.50341796875 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 50.78514099121094, "loadAverage": [ 3.26611328125, 5.328125, 6.482421875 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 52.89529585838318, "loadAverage": [ 3.26611328125, 5.328125, 6.482421875 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 55.01364874839783, "loadAverage": [ 3.24462890625, 5.2890625, 6.4619140625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 57.14417362213135, "loadAverage": [ 3.24462890625, 5.2890625, 6.4619140625 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 59.29050970077515, "loadAverage": [ 3.24462890625, 5.2890625, 6.4619140625 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 61.41256880760193, "loadAverage": [ 3.14453125, 5.234375, 6.435546875 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 63.53537368774414, "loadAverage": [ 3.14453125, 5.234375, 6.435546875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 65.65082287788391, "loadAverage": [ 3.052734375, 5.1806640625, 6.4091796875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 67.76627564430237, "loadAverage": [ 3.052734375, 5.1806640625, 6.4091796875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 69.88871598243713, "loadAverage": [ 3.12841796875, 5.16064453125, 6.39501953125 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 72.02046585083008, "loadAverage": [ 3.12841796875, 5.16064453125, 6.39501953125 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 74.21225380897522, "loadAverage": [ 3.12841796875, 5.16064453125, 6.39501953125 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 76.91539096832275, "loadAverage": [ 3.35791015625, 5.17431640625, 6.392578125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 79.14467287063599, "loadAverage": [ 3.35791015625, 5.17431640625, 6.392578125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 81.30665493011475, "loadAverage": [ 3.4091796875, 5.15478515625, 6.37841796875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 83.43819284439087, "loadAverage": [ 3.4091796875, 5.15478515625, 6.37841796875 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 85.605299949646, "loadAverage": [ 3.9365234375, 5.23486328125, 6.3994140625 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 87.86327981948853, "loadAverage": [ 3.9365234375, 5.23486328125, 6.3994140625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 90.07246899604797, "loadAverage": [ 4.58203125, 5.34716796875, 6.43212890625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 92.35756492614746, "loadAverage": [ 4.58203125, 5.34716796875, 6.43212890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 94.55609083175659, "loadAverage": [ 4.58203125, 5.34716796875, 6.43212890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 96.70883870124817, "loadAverage": [ 4.6953125, 5.35791015625, 6.42919921875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 98.86573791503906, "loadAverage": [ 4.6953125, 5.35791015625, 6.42919921875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 101.02082991600037, "loadAverage": [ 4.71923828125, 5.3515625, 6.42041015625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 103.16042184829712, "loadAverage": [ 4.71923828125, 5.3515625, 6.42041015625 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 105.80573678016663, "loadAverage": [ 4.74169921875, 5.345703125, 6.41162109375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 108.7969388961792, "loadAverage": [ 4.74169921875, 5.345703125, 6.41162109375 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 111.00913381576538, "loadAverage": [ 6.20361328125, 5.638671875, 6.5087890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 113.20657277107239, "loadAverage": [ 6.20361328125, 5.638671875, 6.5087890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 115.35015988349915, "loadAverage": [ 5.78662109375, 5.5615234375, 6.47607421875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 117.49540281295776, "loadAverage": [ 5.78662109375, 5.5615234375, 6.47607421875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 119.62505865097046, "loadAverage": [ 5.78662109375, 5.5615234375, 6.47607421875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 121.7389669418335, "loadAverage": [ 5.80322265625, 5.568359375, 6.47314453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 123.87202572822571, "loadAverage": [ 5.80322265625, 5.568359375, 6.47314453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 125.99005270004272, "loadAverage": [ 5.73876953125, 5.55859375, 6.46435546875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 128.10197496414185, "loadAverage": [ 5.73876953125, 5.55859375, 6.46435546875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 130.22027802467346, "loadAverage": [ 5.51904296875, 5.51611328125, 6.44384765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 132.33953261375427, "loadAverage": [ 5.51904296875, 5.51611328125, 6.44384765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 134.45818781852722, "loadAverage": [ 5.51904296875, 5.51611328125, 6.44384765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 136.58158373832703, "loadAverage": [ 5.71728515625, 5.55712890625, 6.45263671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 138.7027838230133, "loadAverage": [ 5.71728515625, 5.55712890625, 6.45263671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 140.85735988616943, "loadAverage": [ 5.97998046875, 5.6142578125, 6.46728515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 142.98833465576172, "loadAverage": [ 5.97998046875, 5.6142578125, 6.46728515625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 145.1077697277069, "loadAverage": [ 6.3818359375, 5.70361328125, 6.49365234375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 147.29294681549072, "loadAverage": [ 6.3818359375, 5.70361328125, 6.49365234375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15827206144, "processSeconds": 149.49549794197083, "profiled": false, "prompt": "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624172.7486222, "uncontended": false, "workload": "chat" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.8913936343538604 }, "endToEndSpeedups": { "adaptive": 1.8296950936715874 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5, "acceptedDraftTokens": 157, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 314, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 12.370083928108215, "decodeTokensPerSecond": 20.695089983851926, "elapsedSeconds": 13.723363041877747, "endToEndTokensPerSecond": 18.654319587611223, "expectedMTPActive": true, "firstTokenSeconds": 0.00038492679595947266, "generatedTokens": 256, "loadSeconds": 5.793571472167969e-05, "name": "adaptive", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "adaptive", "prefillSeconds": 1.3501660823822021, "prefillTokensPerSecond": 71.84301343791381, "promptTokens": 97, "residentMemoryAfterBytes": 15826944000, "residentMemoryBeforeBytes": 15826763776, "ttftSeconds": 1.3506089448928833 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 23.396697998046875, "decodeTokensPerSecond": 10.941714938636663, "elapsedSeconds": 25.109570026397705, "endToEndTokensPerSecond": 10.195315958451978, "expectedMTPActive": false, "firstTokenSeconds": 0.08413410186767578, "generatedTokens": 256, "loadSeconds": 0.00012099742889404297, "name": "baseline", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "disabled", "prefillSeconds": 1.707082986831665, "prefillTokensPerSecond": 56.82207645923024, "promptTokens": 97, "residentMemoryAfterBytes": 15424700416, "residentMemoryBeforeBytes": 15424585728, "ttftSeconds": 1.7913380861282349 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.776184549280887 }, "endToEndSpeedups": { "adaptive": 1.7247195170497995 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5, "acceptedDraftTokens": 157, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 314, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 12.231046080589294, "decodeTokensPerSecond": 20.9303438408488, "elapsedSeconds": 13.762189030647278, "endToEndTokensPerSecond": 18.601691884184177, "expectedMTPActive": true, "firstTokenSeconds": 0.00037801265716552734, "generatedTokens": 256, "loadSeconds": 3.600120544433594e-05, "name": "adaptive", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "adaptive", "prefillSeconds": 1.5273250341415405, "prefillTokensPerSecond": 63.50972964606746, "promptTokens": 97, "residentMemoryAfterBytes": 15827156992, "residentMemoryBeforeBytes": 15827042304, "ttftSeconds": 1.5277390480041504 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.724595069885254, "decodeTokensPerSecond": 11.783879017145342, "elapsedSeconds": 23.735916018486023, "endToEndTokensPerSecond": 10.785343182063077, "expectedMTPActive": false, "firstTokenSeconds": 0.32595109939575195, "generatedTokens": 256, "loadSeconds": 4.601478576660156e-05, "name": "baseline", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "disabled", "prefillSeconds": 2.0090110301971436, "prefillTokensPerSecond": 48.28246263559908, "promptTokens": 97, "residentMemoryAfterBytes": 15424716800, "residentMemoryBeforeBytes": 15424700416, "ttftSeconds": 2.335008144378662 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.676130533389858 }, "endToEndSpeedups": { "adaptive": 1.6225488091370728 }, "greedyOutputParity": true, "promptCharacters": 413, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5, "acceptedDraftTokens": 157, "draftHistoryTokens": 96, "draftModel": "qwen-mtp", "draftedTokens": 314, "rounds": 99, "route": "mtp-speculative" }, "decodeSeconds": 11.994428038597107, "decodeTokensPerSecond": 21.343243644149812, "elapsedSeconds": 13.437964916229248, "endToEndTokensPerSecond": 19.050503673426373, "expectedMTPActive": true, "firstTokenSeconds": 0.0003629922866821289, "generatedTokens": 256, "loadSeconds": 4.208087921142578e-05, "name": "adaptive", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "adaptive", "prefillSeconds": 1.440614938735962, "prefillTokensPerSecond": 67.33235744807052, "promptTokens": 97, "residentMemoryAfterBytes": 15827206144, "residentMemoryBeforeBytes": 15827156992, "ttftSeconds": 1.4410200119018555 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 20.10422706604004, "decodeTokensPerSecond": 12.733640500531052, "elapsedSeconds": 21.803753972053528, "endToEndTokensPerSecond": 11.741097442583614, "expectedMTPActive": false, "firstTokenSeconds": 0.06784498691558838, "generatedTokens": 256, "loadSeconds": 2.6106834411621094e-05, "name": "baseline", "outputSHA256": "c0587748994cafccd1ad8141f9498b0b4764bbffa861c938d49541f2b38ec781", "policy": "disabled", "prefillSeconds": 1.697072982788086, "prefillTokensPerSecond": 57.157235418738864, "promptTokens": 97, "residentMemoryAfterBytes": 15424733184, "residentMemoryBeforeBytes": 15424716800, "ttftSeconds": 1.764944076538086 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/qwen-code", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "f00a0b29aa697cadb975f010c20de531cb9a49f50f7d9ec9ef26a4600a9091b2", "receipt": { "after": { "gpuDeviceUtilizationPercent": 84, "loadAverage": [ 4.10595703125, 6.13623046875, 6.8701171875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 0, "loadAverage": [ 4.66650390625, 7.09130859375, 7.27392578125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 30, "elapsedSeconds": 0.09901285171508789, "loadAverage": [ 4.66650390625, 7.09130859375, 7.27392578125 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 2.2083797454833984, "loadAverage": [ 4.66650390625, 7.09130859375, 7.27392578125 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 4.360245943069458, "loadAverage": [ 4.45263671875, 7.00634765625, 7.24267578125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 6.486861944198608, "loadAverage": [ 4.45263671875, 7.00634765625, 7.24267578125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 8.591266870498657, "loadAverage": [ 4.49609375, 6.97265625, 7.2294921875 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 10.706994771957397, "loadAverage": [ 4.49609375, 6.97265625, 7.2294921875 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 12.825426816940308, "loadAverage": [ 4.49609375, 6.97265625, 7.2294921875 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 14.97171401977539, "loadAverage": [ 4.4560546875, 6.9228515625, 7.21044921875 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 17.09374165534973, "loadAverage": [ 4.4560546875, 6.9228515625, 7.21044921875 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 19.213916778564453, "loadAverage": [ 4.25927734375, 6.8408203125, 7.1796875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 21.340367794036865, "loadAverage": [ 4.25927734375, 6.8408203125, 7.1796875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 23.457843780517578, "loadAverage": [ 4.638671875, 6.87646484375, 7.18994140625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 25.567312717437744, "loadAverage": [ 4.638671875, 6.87646484375, 7.18994140625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 27.688270092010498, "loadAverage": [ 4.638671875, 6.87646484375, 7.18994140625 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 29.8018798828125, "loadAverage": [ 4.66748046875, 6.84521484375, 7.1767578125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 31.92505168914795, "loadAverage": [ 4.66748046875, 6.84521484375, 7.1767578125 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 34.04775881767273, "loadAverage": [ 5.09423828125, 6.8974609375, 7.19287109375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 36.17003893852234, "loadAverage": [ 5.09423828125, 6.8974609375, 7.19287109375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 38.297491788864136, "loadAverage": [ 5.24658203125, 6.89892578125, 7.19140625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 40.418861627578735, "loadAverage": [ 5.24658203125, 6.89892578125, 7.19140625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 42.53457474708557, "loadAverage": [ 5.24658203125, 6.89892578125, 7.19140625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 44.65687274932861, "loadAverage": [ 5.2265625, 6.8671875, 7.17822265625 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 46.76511478424072, "loadAverage": [ 5.2265625, 6.8671875, 7.17822265625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 48.8657808303833, "loadAverage": [ 5.5283203125, 6.90234375, 7.1884765625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 50.98022699356079, "loadAverage": [ 5.5283203125, 6.90234375, 7.1884765625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 53.08919072151184, "loadAverage": [ 5.5283203125, 6.90234375, 7.1884765625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 55.21355700492859, "loadAverage": [ 5.72607421875, 6.92041015625, 7.19287109375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 57.33139181137085, "loadAverage": [ 5.72607421875, 6.92041015625, 7.19287109375 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 59.44509696960449, "loadAverage": [ 5.58740234375, 6.87158203125, 7.173828125 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 61.56398892402649, "loadAverage": [ 5.58740234375, 6.87158203125, 7.173828125 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 63.68010687828064, "loadAverage": [ 5.4599609375, 6.82373046875, 7.15478515625 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 65.78901600837708, "loadAverage": [ 5.4599609375, 6.82373046875, 7.15478515625 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 67.91829299926758, "loadAverage": [ 5.4599609375, 6.82373046875, 7.15478515625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 70.03332877159119, "loadAverage": [ 5.3427734375, 6.7763671875, 7.13623046875 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 72.15995693206787, "loadAverage": [ 5.3427734375, 6.7763671875, 7.13623046875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 74.27985978126526, "loadAverage": [ 5.07470703125, 6.69677734375, 7.10595703125 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 76.39697194099426, "loadAverage": [ 5.07470703125, 6.69677734375, 7.10595703125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 78.52141284942627, "loadAverage": [ 4.908203125, 6.63525390625, 7.08154296875 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 80.63940501213074, "loadAverage": [ 4.908203125, 6.63525390625, 7.08154296875 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 82.75234484672546, "loadAverage": [ 4.908203125, 6.63525390625, 7.08154296875 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 84.86209464073181, "loadAverage": [ 4.99560546875, 6.62451171875, 7.0751953125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 86.97828984260559, "loadAverage": [ 4.99560546875, 6.62451171875, 7.0751953125 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 89.09441685676575, "loadAverage": [ 4.67529296875, 6.53076171875, 7.03955078125 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 91.19868993759155, "loadAverage": [ 4.67529296875, 6.53076171875, 7.03955078125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 93.32188177108765, "loadAverage": [ 4.701171875, 6.5048828125, 7.02734375 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 95.43754386901855, "loadAverage": [ 4.701171875, 6.5048828125, 7.02734375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 97.54706883430481, "loadAverage": [ 4.701171875, 6.5048828125, 7.02734375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 99.650869846344, "loadAverage": [ 4.40478515625, 6.4130859375, 6.99169921875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 101.76723575592041, "loadAverage": [ 4.40478515625, 6.4130859375, 6.99169921875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 103.88547992706299, "loadAverage": [ 4.1318359375, 6.32275390625, 6.95654296875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 105.99157094955444, "loadAverage": [ 4.1318359375, 6.32275390625, 6.95654296875 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 108.10663390159607, "loadAverage": [ 4.1318359375, 6.32275390625, 6.95654296875 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 110.22886490821838, "loadAverage": [ 3.9609375, 6.2509765625, 6.92724609375 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 112.34249067306519, "loadAverage": [ 3.9609375, 6.2509765625, 6.92724609375 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 114.46061086654663, "loadAverage": [ 3.9638671875, 6.21337890625, 6.90966796875 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 116.56583499908447, "loadAverage": [ 3.9638671875, 6.21337890625, 6.90966796875 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 118.67221879959106, "loadAverage": [ 4.04638671875, 6.19287109375, 6.8984375 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 120.78474283218384, "loadAverage": [ 4.04638671875, 6.19287109375, 6.8984375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 122.89125871658325, "loadAverage": [ 4.04638671875, 6.19287109375, 6.8984375 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 124.99504089355469, "loadAverage": [ 4.20263671875, 6.189453125, 6.89306640625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 127.11200284957886, "loadAverage": [ 4.20263671875, 6.189453125, 6.89306640625 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 129.23091292381287, "loadAverage": [ 4.10595703125, 6.13623046875, 6.8701171875 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15830417408, "processSeconds": 131.32760977745056, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624004.1133142, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.5224490637272377 }, "endToEndSpeedups": { "adaptive": 2.4369467935271687 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.443959951400757, "decodeTokensPerSecond": 11.93809355082654, "elapsedSeconds": 22.58056604862213, "endToEndTokensPerSecond": 11.337182577653811, "expectedMTPActive": false, "firstTokenSeconds": 0.0707850456237793, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 1.1344571113586426, "prefillTokensPerSecond": 47.59986028500346, "promptTokens": 54, "residentMemoryAfterBytes": 15397060608, "residentMemoryBeforeBytes": 15396929536, "ttftSeconds": 1.205268144607544 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 8.501245975494385, "decodeTokensPerSecond": 30.11323289997058, "elapsedSeconds": 9.26592493057251, "endToEndTokensPerSecond": 27.628110730245538, "expectedMTPActive": true, "firstTokenSeconds": 0.0002989768981933594, "generatedTokens": 256, "loadSeconds": 2.09808349609375e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.7626460790634155, "prefillTokensPerSecond": 70.80610716089421, "promptTokens": 54, "residentMemoryAfterBytes": 15829991424, "residentMemoryBeforeBytes": 15829893120, "ttftSeconds": 0.7629660367965698 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.375324440852814 }, "endToEndSpeedups": { "adaptive": 2.311770912662055 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.099429965019226, "decodeTokensPerSecond": 12.133029206211862, "elapsedSeconds": 22.095813989639282, "endToEndTokensPerSecond": 11.585904919367907, "expectedMTPActive": false, "firstTokenSeconds": 0.059158921241760254, "generatedTokens": 256, "loadSeconds": 2.09808349609375e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.9940139055252075, "prefillTokensPerSecond": 54.325195754145916, "promptTokens": 54, "residentMemoryAfterBytes": 15397289984, "residentMemoryBeforeBytes": 15397175296, "ttftSeconds": 1.0531938076019287 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 8.882757067680359, "decodeTokensPerSecond": 28.81988081509605, "elapsedSeconds": 9.557960033416748, "endToEndTokensPerSecond": 26.78395798946294, "expectedMTPActive": true, "firstTokenSeconds": 0.00045096874237060547, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.6724839210510254, "prefillTokensPerSecond": 80.29931766339243, "promptTokens": 54, "residentMemoryAfterBytes": 15830056960, "residentMemoryBeforeBytes": 15829991424, "ttftSeconds": 0.6729608774185181 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.271887326268773 }, "endToEndSpeedups": { "adaptive": 2.2170458263220043 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 20.500563979148865, "decodeTokensPerSecond": 12.487461333277356, "elapsedSeconds": 21.570683002471924, "endToEndTokensPerSecond": 11.867959858789048, "expectedMTPActive": false, "firstTokenSeconds": 0.06422305107116699, "generatedTokens": 256, "loadSeconds": 2.5033950805664062e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 1.0670969486236572, "prefillTokensPerSecond": 50.60458664945978, "promptTokens": 54, "residentMemoryAfterBytes": 15397552128, "residentMemoryBeforeBytes": 15397289984, "ttftSeconds": 1.1313450336456299 }, { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 9.023583054542542, "decodeTokensPerSecond": 28.37010514034418, "elapsedSeconds": 9.72947096824646, "endToEndTokensPerSecond": 26.311810871885342, "expectedMTPActive": true, "firstTokenSeconds": 0.0004260540008544922, "generatedTokens": 256, "loadSeconds": 6.198883056640625e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.7028970718383789, "prefillTokensPerSecond": 76.82490390629557, "promptTokens": 54, "residentMemoryAfterBytes": 15830384640, "residentMemoryBeforeBytes": 15830056960, "ttftSeconds": 0.7033851146697998 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/qwen-math", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "7323b39000a65bfdc76c74abcddf2aae36ad8d67ec63852ad30a6350ddfc268e", "receipt": { "after": { "gpuDeviceUtilizationPercent": 64, "loadAverage": [ 6.49072265625, 5.58837890625, 6.125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 57, "loadAverage": [ 5.79248046875, 5.6796875, 6.28564453125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Derive the sum of the first n squares using induction. Explain the base case and inductive step, then check your formula for n=10. Show each algebraic step.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 61, "elapsedSeconds": 0.13121700286865234, "loadAverage": [ 5.79248046875, 5.6796875, 6.28564453125 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 2.242560863494873, "loadAverage": [ 6.04931640625, 5.734375, 6.30126953125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 4.336018085479736, "loadAverage": [ 6.04931640625, 5.734375, 6.30126953125 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 6.433802843093872, "loadAverage": [ 6.044921875, 5.73876953125, 6.29931640625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 8.533766031265259, "loadAverage": [ 6.044921875, 5.73876953125, 6.29931640625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 10.627536058425903, "loadAverage": [ 5.640625, 5.65966796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 12.720891952514648, "loadAverage": [ 5.640625, 5.65966796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 14.816335916519165, "loadAverage": [ 5.640625, 5.65966796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 16.91425085067749, "loadAverage": [ 5.9091796875, 5.71484375, 6.28369140625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 19.0136821269989, "loadAverage": [ 5.9091796875, 5.71484375, 6.28369140625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 21.111894845962524, "loadAverage": [ 5.755859375, 5.68603515625, 6.27001953125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 23.21132779121399, "loadAverage": [ 5.755859375, 5.68603515625, 6.27001953125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 25.307176113128662, "loadAverage": [ 5.775390625, 5.69091796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 27.403681993484497, "loadAverage": [ 5.775390625, 5.69091796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 29.501554012298584, "loadAverage": [ 5.775390625, 5.69091796875, 6.26806640625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 31.597473859786987, "loadAverage": [ 5.6328125, 5.66259765625, 6.25439453125 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 33.6899847984314, "loadAverage": [ 5.6328125, 5.66259765625, 6.25439453125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 35.78790903091431, "loadAverage": [ 5.421875, 5.6181640625, 6.23486328125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 37.8883581161499, "loadAverage": [ 5.421875, 5.6181640625, 6.23486328125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 39.984498023986816, "loadAverage": [ 5.421875, 5.6181640625, 6.23486328125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 42.080820083618164, "loadAverage": [ 5.2275390625, 5.57421875, 6.2158203125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 44.17714500427246, "loadAverage": [ 5.2275390625, 5.57421875, 6.2158203125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 46.27063703536987, "loadAverage": [ 4.96875, 5.5146484375, 6.19091796875 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 48.364442110061646, "loadAverage": [ 4.96875, 5.5146484375, 6.19091796875 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 50.49424600601196, "loadAverage": [ 4.97119140625, 5.505859375, 6.18359375 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 52.61847114562988, "loadAverage": [ 4.97119140625, 5.505859375, 6.18359375 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 54.73383116722107, "loadAverage": [ 4.97119140625, 5.505859375, 6.18359375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 56.844104051589966, "loadAverage": [ 4.89306640625, 5.48046875, 6.17041015625 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 58.96553683280945, "loadAverage": [ 4.89306640625, 5.48046875, 6.17041015625 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 61.07891917228699, "loadAverage": [ 4.6611328125, 5.42236328125, 6.1455078125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 63.1900200843811, "loadAverage": [ 4.6611328125, 5.42236328125, 6.1455078125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 65.30997395515442, "loadAverage": [ 4.44775390625, 5.365234375, 6.12109375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 67.41373896598816, "loadAverage": [ 4.44775390625, 5.365234375, 6.12109375 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 69.53113102912903, "loadAverage": [ 4.44775390625, 5.365234375, 6.12109375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 71.63934206962585, "loadAverage": [ 4.41162109375, 5.34228515625, 6.1083984375 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 73.75040292739868, "loadAverage": [ 4.41162109375, 5.34228515625, 6.1083984375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 75.85357403755188, "loadAverage": [ 4.13818359375, 5.27001953125, 6.078125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 77.95282506942749, "loadAverage": [ 4.13818359375, 5.27001953125, 6.078125 ] }, { "deviceUtilizationPercent": 50, "elapsedSeconds": 80.10093808174133, "loadAverage": [ 4.13818359375, 5.27001953125, 6.078125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 82.22709894180298, "loadAverage": [ 4.126953125, 5.24853515625, 6.06591796875 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 84.34500288963318, "loadAverage": [ 4.126953125, 5.24853515625, 6.06591796875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 86.45307779312134, "loadAverage": [ 4.43701171875, 5.2939453125, 6.0771484375 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 88.56861400604248, "loadAverage": [ 4.43701171875, 5.2939453125, 6.0771484375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 90.67175197601318, "loadAverage": [ 4.48193359375, 5.2890625, 6.07080078125 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 92.79061007499695, "loadAverage": [ 4.48193359375, 5.2890625, 6.07080078125 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 94.89083003997803, "loadAverage": [ 4.48193359375, 5.2890625, 6.07080078125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 96.99864101409912, "loadAverage": [ 4.36279296875, 5.2509765625, 6.052734375 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 99.11559891700745, "loadAverage": [ 4.36279296875, 5.2509765625, 6.052734375 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 101.21855807304382, "loadAverage": [ 4.57373046875, 5.27978515625, 6.05810546875 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 103.33095908164978, "loadAverage": [ 4.57373046875, 5.27978515625, 6.05810546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 105.4318299293518, "loadAverage": [ 4.767578125, 5.30810546875, 6.0634765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 107.53883790969849, "loadAverage": [ 4.767578125, 5.30810546875, 6.0634765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 109.65546607971191, "loadAverage": [ 4.767578125, 5.30810546875, 6.0634765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 111.75772500038147, "loadAverage": [ 4.7060546875, 5.2861328125, 6.05126953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 113.86665415763855, "loadAverage": [ 4.7060546875, 5.2861328125, 6.05126953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 115.97425603866577, "loadAverage": [ 4.6494140625, 5.2646484375, 6.0390625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 118.0734920501709, "loadAverage": [ 4.6494140625, 5.2646484375, 6.0390625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 120.18933892250061, "loadAverage": [ 4.6494140625, 5.2646484375, 6.0390625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 122.29121398925781, "loadAverage": [ 4.51708984375, 5.22705078125, 6.02099609375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 124.39167094230652, "loadAverage": [ 4.51708984375, 5.22705078125, 6.02099609375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 126.48527097702026, "loadAverage": [ 4.5556640625, 5.22314453125, 6.0146484375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 128.58461093902588, "loadAverage": [ 4.5556640625, 5.22314453125, 6.0146484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 130.6853301525116, "loadAverage": [ 4.2705078125, 5.15283203125, 5.98486328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 132.78867101669312, "loadAverage": [ 4.2705078125, 5.15283203125, 5.98486328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 134.88951301574707, "loadAverage": [ 4.2705078125, 5.15283203125, 5.98486328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 137.0003170967102, "loadAverage": [ 4.24853515625, 5.13330078125, 5.97314453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 139.10114097595215, "loadAverage": [ 4.24853515625, 5.13330078125, 5.97314453125 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 141.6176369190216, "loadAverage": [ 5.75, 5.4296875, 6.07275390625 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 144.51293301582336, "loadAverage": [ 5.75, 5.4296875, 6.07275390625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 147.306391954422, "loadAverage": [ 6.49072265625, 5.58837890625, 6.125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15807676416, "processSeconds": 149.56943202018738, "profiled": false, "prompt": "Derive the sum of the first n squares using induction. Explain the base case and inductive step, then check your formula for n=10. Show each algebraic step.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624547.208699, "uncontended": false, "workload": "math" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.3530700637421837 }, "endToEndSpeedups": { "adaptive": 2.283659621683949 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.706959706959707, "acceptedDraftTokens": 193, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 273, "rounds": 63, "route": "mtp-speculative" }, "decodeSeconds": 10.775597095489502, "decodeTokensPerSecond": 23.757384183114794, "elapsedSeconds": 11.670462965965271, "endToEndTokensPerSecond": 21.935719323781434, "expectedMTPActive": true, "firstTokenSeconds": 0.0006320476531982422, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "adaptive", "prefillSeconds": 0.8934379816055298, "prefillTokensPerSecond": 54.84432160802679, "promptTokens": 49, "residentMemoryAfterBytes": 15807299584, "residentMemoryBeforeBytes": 15807021056, "ttftSeconds": 0.8940900564193726 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 25.355734944343567, "decodeTokensPerSecond": 10.096335229955905, "elapsedSeconds": 26.651365041732788, "endToEndTokensPerSecond": 9.605511747677285, "expectedMTPActive": false, "firstTokenSeconds": 0.13247692584991455, "generatedTokens": 256, "loadSeconds": 4.398822784423828e-05, "name": "baseline", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "disabled", "prefillSeconds": 1.2924760580062866, "prefillTokensPerSecond": 37.91172741380225, "promptTokens": 49, "residentMemoryAfterBytes": 15428976640, "residentMemoryBeforeBytes": 15428911104, "ttftSeconds": 1.4249969720840454 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.137897091686745 }, "endToEndSpeedups": { "adaptive": 2.083269417935927 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.706959706959707, "acceptedDraftTokens": 193, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 273, "rounds": 63, "route": "mtp-speculative" }, "decodeSeconds": 10.71203899383545, "decodeTokensPerSecond": 23.898344670638576, "elapsedSeconds": 11.53477692604065, "endToEndTokensPerSecond": 22.193753866367388, "expectedMTPActive": true, "firstTokenSeconds": 0.0002900362014770508, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "adaptive", "prefillSeconds": 0.8212320804595947, "prefillTokensPerSecond": 59.666446508735525, "promptTokens": 49, "residentMemoryAfterBytes": 15807594496, "residentMemoryBeforeBytes": 15807414272, "ttftSeconds": 0.8215421438217163 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 22.90123701095581, "decodeTokensPerSecond": 11.178435465190425, "elapsedSeconds": 24.03004801273346, "endToEndTokensPerSecond": 10.653328693490177, "expectedMTPActive": false, "firstTokenSeconds": 0.07827401161193848, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "baseline", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "disabled", "prefillSeconds": 1.1273119449615479, "prefillTokensPerSecond": 43.46622975033886, "promptTokens": 49, "residentMemoryAfterBytes": 15429025792, "residentMemoryBeforeBytes": 15428993024, "ttftSeconds": 1.2056119441986084 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.024663030502862 }, "endToEndSpeedups": { "adaptive": 1.9775546966261714 }, "greedyOutputParity": true, "promptCharacters": 156, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.706959706959707, "acceptedDraftTokens": 193, "draftHistoryTokens": 48, "draftModel": "qwen-mtp", "draftedTokens": 273, "rounds": 63, "route": "mtp-speculative" }, "decodeSeconds": 10.782731056213379, "decodeTokensPerSecond": 23.74166606450636, "elapsedSeconds": 11.638966083526611, "endToEndTokensPerSecond": 21.995080848489927, "expectedMTPActive": true, "firstTokenSeconds": 0.0002969503402709961, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "adaptive", "prefillSeconds": 0.8548060655593872, "prefillTokensPerSecond": 57.32294373453501, "promptTokens": 49, "residentMemoryAfterBytes": 15807643648, "residentMemoryBeforeBytes": 15807594496, "ttftSeconds": 0.8551230430603027 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.8313969373703, "decodeTokensPerSecond": 11.726230837834624, "elapsedSeconds": 23.01669204235077, "endToEndTokensPerSecond": 11.12236282820135, "expectedMTPActive": false, "firstTokenSeconds": 0.06294500827789307, "generatedTokens": 256, "loadSeconds": 2.193450927734375e-05, "name": "baseline", "outputSHA256": "e89711cb589e92df06a8ff48eace2ae459591085ced5c2da40b212eec5cd4fd5", "policy": "disabled", "prefillSeconds": 1.1814539432525635, "prefillTokensPerSecond": 41.474320924523, "promptTokens": 49, "residentMemoryAfterBytes": 15429091328, "residentMemoryBeforeBytes": 15429025792, "ttftSeconds": 1.2444208860397339 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "5e4d9c51333a70a47a45a43ae3587349c73515060beb54ce6c72c370e4eeda32", "receipt": { "after": { "gpuDeviceUtilizationPercent": 61, "loadAverage": [ 5.52734375, 5.56689453125, 6.271484375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 49, "loadAverage": [ 6.3115234375, 5.75732421875, 6.4755859375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 50, "elapsedSeconds": 0.13635516166687012, "loadAverage": [ 6.3115234375, 5.75732421875, 6.4755859375 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 2.2430362701416016, "loadAverage": [ 6.3115234375, 5.75732421875, 6.4755859375 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 4.368925333023071, "loadAverage": [ 6.3115234375, 5.75732421875, 6.4755859375 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 6.484771251678467, "loadAverage": [ 6.2861328125, 5.76123046875, 6.47265625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 8.598513126373291, "loadAverage": [ 6.2861328125, 5.76123046875, 6.47265625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 10.707463264465332, "loadAverage": [ 6.02294921875, 5.71533203125, 6.4521484375 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 12.824186086654663, "loadAverage": [ 6.02294921875, 5.71533203125, 6.4521484375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 14.935338020324707, "loadAverage": [ 5.86083984375, 5.6865234375, 6.4375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 17.055726051330566, "loadAverage": [ 5.86083984375, 5.6865234375, 6.4375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 19.176573991775513, "loadAverage": [ 5.86083984375, 5.6865234375, 6.4375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 21.28590726852417, "loadAverage": [ 5.95166015625, 5.7080078125, 6.4404296875 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 23.393944025039673, "loadAverage": [ 5.95166015625, 5.7080078125, 6.4404296875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 25.497557163238525, "loadAverage": [ 5.71484375, 5.66259765625, 6.419921875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 27.60605001449585, "loadAverage": [ 5.71484375, 5.66259765625, 6.419921875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 29.714590072631836, "loadAverage": [ 5.4169921875, 5.6015625, 6.3935546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 31.828978300094604, "loadAverage": [ 5.4169921875, 5.6015625, 6.3935546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 33.98128628730774, "loadAverage": [ 5.4169921875, 5.6015625, 6.3935546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 36.10975909233093, "loadAverage": [ 5.46337890625, 5.60791015625, 6.39111328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 38.22799015045166, "loadAverage": [ 5.46337890625, 5.60791015625, 6.39111328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 40.36394214630127, "loadAverage": [ 5.26611328125, 5.564453125, 6.37109375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 42.473140001297, "loadAverage": [ 5.26611328125, 5.564453125, 6.37109375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 44.57291007041931, "loadAverage": [ 5.26611328125, 5.564453125, 6.37109375 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 46.693432092666626, "loadAverage": [ 5.40478515625, 5.587890625, 6.37451171875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 48.82114005088806, "loadAverage": [ 5.40478515625, 5.587890625, 6.37451171875 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 50.941697120666504, "loadAverage": [ 5.9326171875, 5.6943359375, 6.4072265625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 53.05119204521179, "loadAverage": [ 5.9326171875, 5.6943359375, 6.4072265625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 55.16583299636841, "loadAverage": [ 6.09814453125, 5.732421875, 6.41650390625 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 57.28836727142334, "loadAverage": [ 6.09814453125, 5.732421875, 6.41650390625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 59.4020471572876, "loadAverage": [ 6.09814453125, 5.732421875, 6.41650390625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 61.50659203529358, "loadAverage": [ 6.009765625, 5.72021484375, 6.408203125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 63.61092400550842, "loadAverage": [ 6.009765625, 5.72021484375, 6.408203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 65.71856713294983, "loadAverage": [ 5.6083984375, 5.6416015625, 6.37646484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 67.82051515579224, "loadAverage": [ 5.6083984375, 5.6416015625, 6.37646484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 69.93814826011658, "loadAverage": [ 5.2392578125, 5.564453125, 6.3447265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 72.04796600341797, "loadAverage": [ 5.2392578125, 5.564453125, 6.3447265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 74.1594340801239, "loadAverage": [ 5.2392578125, 5.564453125, 6.3447265625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 76.27951121330261, "loadAverage": [ 4.9794921875, 5.5048828125, 6.31884765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 78.39089822769165, "loadAverage": [ 4.9794921875, 5.5048828125, 6.31884765625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 80.51183795928955, "loadAverage": [ 5.14111328125, 5.529296875, 6.32275390625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 82.61635613441467, "loadAverage": [ 5.14111328125, 5.529296875, 6.32275390625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 84.73375916481018, "loadAverage": [ 4.96923828125, 5.4873046875, 6.30322265625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 86.84868931770325, "loadAverage": [ 4.96923828125, 5.4873046875, 6.30322265625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 88.94682812690735, "loadAverage": [ 4.96923828125, 5.4873046875, 6.30322265625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 91.04559016227722, "loadAverage": [ 4.8115234375, 5.44580078125, 6.28369140625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 93.14368104934692, "loadAverage": [ 4.8115234375, 5.44580078125, 6.28369140625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 95.2387101650238, "loadAverage": [ 4.826171875, 5.43798828125, 6.27587890625 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 97.34384417533875, "loadAverage": [ 4.826171875, 5.43798828125, 6.27587890625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 99.44666504859924, "loadAverage": [ 4.826171875, 5.43798828125, 6.27587890625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 101.54118418693542, "loadAverage": [ 5.0, 5.4638671875, 6.27978515625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 103.64293193817139, "loadAverage": [ 5.0, 5.4638671875, 6.27978515625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 105.74261212348938, "loadAverage": [ 4.83984375, 5.4228515625, 6.26025390625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 107.85158205032349, "loadAverage": [ 4.83984375, 5.4228515625, 6.26025390625 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 109.95608401298523, "loadAverage": [ 5.1728515625, 5.48193359375, 6.27587890625 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 112.06396698951721, "loadAverage": [ 5.1728515625, 5.48193359375, 6.27587890625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 114.17619824409485, "loadAverage": [ 5.1728515625, 5.48193359375, 6.27587890625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 116.2763421535492, "loadAverage": [ 5.39892578125, 5.5234375, 6.28564453125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 118.37764811515808, "loadAverage": [ 5.39892578125, 5.5234375, 6.28564453125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 120.48455619812012, "loadAverage": [ 6.00732421875, 5.6474609375, 6.32470703125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 122.58062410354614, "loadAverage": [ 6.00732421875, 5.6474609375, 6.32470703125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 124.6838002204895, "loadAverage": [ 6.00732421875, 5.6474609375, 6.32470703125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 126.78776121139526, "loadAverage": [ 6.40673828125, 5.73583984375, 6.35205078125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 128.88950300216675, "loadAverage": [ 6.40673828125, 5.73583984375, 6.35205078125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 130.99153900146484, "loadAverage": [ 5.8935546875, 5.64013671875, 6.314453125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 133.08962607383728, "loadAverage": [ 5.8935546875, 5.64013671875, 6.314453125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 135.18944001197815, "loadAverage": [ 5.74169921875, 5.61279296875, 6.30078125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 137.29600620269775, "loadAverage": [ 5.74169921875, 5.61279296875, 6.30078125 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 139.45655512809753, "loadAverage": [ 5.74169921875, 5.61279296875, 6.30078125 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 141.56350016593933, "loadAverage": [ 5.44189453125, 5.552734375, 6.275390625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 143.66803812980652, "loadAverage": [ 5.44189453125, 5.552734375, 6.275390625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 145.76345920562744, "loadAverage": [ 5.486328125, 5.56005859375, 6.2734375 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 147.86328125, "loadAverage": [ 5.486328125, 5.56005859375, 6.2734375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 149.9568099975586, "loadAverage": [ 5.52734375, 5.56689453125, 6.271484375 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 152.05420422554016, "loadAverage": [ 5.52734375, 5.56689453125, 6.271484375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15824977920, "processSeconds": 154.14823722839355, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624357.6931489, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.2150808420673067 }, "endToEndSpeedups": { "adaptive": 1.215459090113719 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 19.4006769657135, "decodeTokensPerSecond": 13.19541583277865, "elapsedSeconds": 20.4760639667511, "endToEndTokensPerSecond": 12.50240282584051, "expectedMTPActive": false, "firstTokenSeconds": 0.06372296810150146, "generatedTokens": 256, "loadSeconds": 2.396106719970703e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 1.073531985282898, "prefillTokensPerSecond": 55.890276975947536, "promptTokens": 60, "residentMemoryAfterBytes": 15414591488, "residentMemoryBeforeBytes": 15414509568, "ttftSeconds": 1.1372789144515991 }, { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 15.966572999954224, "decodeTokensPerSecond": 16.033496981520955, "elapsedSeconds": 16.846362113952637, "endToEndTokensPerSecond": 15.196159162931297, "expectedMTPActive": true, "firstTokenSeconds": 0.0002650022506713867, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.8781039714813232, "prefillTokensPerSecond": 68.32903841532867, "promptTokens": 60, "residentMemoryAfterBytes": 15824912384, "residentMemoryBeforeBytes": 15824076800, "ttftSeconds": 0.8783890008926392 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0700793948901455 }, "endToEndSpeedups": { "adaptive": 1.0607492434011625 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 19.021972060203552, "decodeTokensPerSecond": 13.45812091352954, "elapsedSeconds": 19.944918990135193, "endToEndTokensPerSecond": 12.835349199794607, "expectedMTPActive": false, "firstTokenSeconds": 0.0533519983291626, "generatedTokens": 256, "loadSeconds": 2.6941299438476562e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.9209740161895752, "prefillTokensPerSecond": 65.14841781122463, "promptTokens": 60, "residentMemoryAfterBytes": 15414837248, "residentMemoryBeforeBytes": 15414706176, "ttftSeconds": 0.9743529558181763 }, { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 17.776224970817566, "decodeTokensPerSecond": 14.4012578835081, "elapsedSeconds": 18.80267095565796, "endToEndTokensPerSecond": 13.615086952471845, "expectedMTPActive": true, "firstTokenSeconds": 0.0005480051040649414, "generatedTokens": 256, "loadSeconds": 2.9087066650390625e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 1.0247420072555542, "prefillTokensPerSecond": 58.551322747752806, "promptTokens": 60, "residentMemoryAfterBytes": 15824928768, "residentMemoryBeforeBytes": 15824928768, "ttftSeconds": 1.0253190994262695 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0922572375419586 }, "endToEndSpeedups": { "adaptive": 1.0961168168568942 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 19.20444905757904, "decodeTokensPerSecond": 13.330244425781615, "elapsedSeconds": 20.29848301410675, "endToEndTokensPerSecond": 12.611779896167057, "expectedMTPActive": false, "firstTokenSeconds": 0.06122004985809326, "generatedTokens": 256, "loadSeconds": 3.409385681152344e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 1.0918149948120117, "prefillTokensPerSecond": 54.95436524054222, "promptTokens": 60, "residentMemoryAfterBytes": 15414870016, "residentMemoryBeforeBytes": 15414837248, "ttftSeconds": 1.1530691385269165 }, { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 17.58235001564026, "decodeTokensPerSecond": 14.560055952263319, "elapsedSeconds": 18.518539905548096, "endToEndTokensPerSecond": 13.823984034686408, "expectedMTPActive": true, "firstTokenSeconds": 0.0004259347915649414, "generatedTokens": 256, "loadSeconds": 2.205371856689453e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.9345439672470093, "prefillTokensPerSecond": 64.20243680642304, "promptTokens": 60, "residentMemoryAfterBytes": 15824945152, "residentMemoryBeforeBytes": 15824928768, "ttftSeconds": 0.9349919557571411 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-greedy/qwen-summary", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "0f40a578a16212e6b5529054a833883e4ab06aaf9ebf3d39140d9376edfb3d15", "receipt": { "after": { "gpuDeviceUtilizationPercent": 87, "loadAverage": [ 5.3916015625, 6.2470703125, 6.35302734375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 59, "loadAverage": [ 8.35107421875, 6.3984375, 6.40625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Summarize this fictional incident report for a manager. Explain the timeline, user impact, cause, recovery, and three concrete follow-up actions. Distinguish confirmed facts from open questions. Use clear paragraphs and a short action list.\n\n09:02: The team deployed version 2.8 of the document search service. 09:07: Monitoring showed the 95th percentile search latency rising from 180 milliseconds to 4.2 seconds. 09:10: Support received six reports of slow searches; document uploads and downloads remained available. 09:13: The on-call engineer paused the rollout at 40 percent of traffic. 09:17: Logs showed repeated cache misses for queries containing a language filter. A new cache key included a randomly generated request identifier. 09:21: The engineer rolled traffic back to version 2.7. 09:26: Latency returned below 250 milliseconds. 09:32: The team confirmed no documents were lost or modified. Approximately 1,800 search requests experienced elevated latency; 73 timed out. The exact number of affected users is still being calculated. The release tests covered result correctness but did not assert cache reuse. A separate warning about database connection count appeared at 09:15; the team has not established whether it was a cause or an effect. Proposed follow-ups include a deterministic cache-key unit test, a staging workload with repeated filtered queries, and an automatic rollback threshold for latency.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 43, "elapsedSeconds": 0.1393280029296875, "loadAverage": [ 8.35107421875, 6.3984375, 6.40625 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 2.2418088912963867, "loadAverage": [ 8.35107421875, 6.3984375, 6.40625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 4.363068103790283, "loadAverage": [ 8.08251953125, 6.375, 6.39794921875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 6.464037895202637, "loadAverage": [ 8.08251953125, 6.375, 6.39794921875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 8.563050031661987, "loadAverage": [ 7.67529296875, 6.31884765625, 6.3779296875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 10.678048849105835, "loadAverage": [ 7.67529296875, 6.31884765625, 6.3779296875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 12.786223888397217, "loadAverage": [ 7.67529296875, 6.31884765625, 6.3779296875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 14.887305974960327, "loadAverage": [ 7.380859375, 6.2802734375, 6.36376953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 17.005151987075806, "loadAverage": [ 7.380859375, 6.2802734375, 6.36376953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 19.108398914337158, "loadAverage": [ 7.27001953125, 6.275390625, 6.361328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 21.21261501312256, "loadAverage": [ 7.27001953125, 6.275390625, 6.361328125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 23.31520915031433, "loadAverage": [ 7.16796875, 6.2705078125, 6.35888671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 25.419042825698853, "loadAverage": [ 7.16796875, 6.2705078125, 6.35888671875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 27.51328182220459, "loadAverage": [ 7.16796875, 6.2705078125, 6.35888671875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 29.61873698234558, "loadAverage": [ 6.75390625, 6.19921875, 6.3330078125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 31.74898910522461, "loadAverage": [ 6.75390625, 6.19921875, 6.3330078125 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 33.86541819572449, "loadAverage": [ 6.29296875, 6.11279296875, 6.3017578125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 35.96995997428894, "loadAverage": [ 6.29296875, 6.11279296875, 6.3017578125 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 38.073760986328125, "loadAverage": [ 6.29296875, 6.11279296875, 6.3017578125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 40.17427396774292, "loadAverage": [ 6.18896484375, 6.09423828125, 6.2939453125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 42.27608108520508, "loadAverage": [ 6.18896484375, 6.09423828125, 6.2939453125 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 44.395549058914185, "loadAverage": [ 6.013671875, 6.05908203125, 6.2802734375 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 46.52087688446045, "loadAverage": [ 6.013671875, 6.05908203125, 6.2802734375 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 48.62930107116699, "loadAverage": [ 5.53173828125, 5.9580078125, 6.2431640625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 50.74443984031677, "loadAverage": [ 5.53173828125, 5.9580078125, 6.2431640625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 52.836498975753784, "loadAverage": [ 5.53173828125, 5.9580078125, 6.2431640625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 54.93623089790344, "loadAverage": [ 5.08837890625, 5.85888671875, 6.20654296875 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 57.76303195953369, "loadAverage": [ 5.08837890625, 5.85888671875, 6.20654296875 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 60.438982009887695, "loadAverage": [ 5.4013671875, 5.91064453125, 6.22265625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 62.988378047943115, "loadAverage": [ 5.4013671875, 5.91064453125, 6.22265625 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 65.85784888267517, "loadAverage": [ 6.41015625, 6.111328125, 6.29150390625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 68.44013977050781, "loadAverage": [ 6.6171875, 6.1591796875, 6.30712890625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 71.55483984947205, "loadAverage": [ 6.6171875, 6.1591796875, 6.30712890625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 74.65407180786133, "loadAverage": [ 8.4091796875, 6.5380859375, 6.43994140625 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 77.62806296348572, "loadAverage": [ 8.4091796875, 6.5380859375, 6.43994140625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 79.8785650730133, "loadAverage": [ 8.9365234375, 6.67822265625, 6.48974609375 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 82.77118992805481, "loadAverage": [ 8.9365234375, 6.67822265625, 6.48974609375 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 86.34701609611511, "loadAverage": [ 8.62109375, 6.64990234375, 6.48095703125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 89.35654091835022, "loadAverage": [ 10.3330078125, 7.037109375, 6.61865234375 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 92.22482109069824, "loadAverage": [ 10.3330078125, 7.037109375, 6.61865234375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 94.44527506828308, "loadAverage": [ 10.14599609375, 7.052734375, 6.62646484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 96.58075881004333, "loadAverage": [ 10.14599609375, 7.052734375, 6.62646484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 98.69560313224792, "loadAverage": [ 9.57373046875, 6.9853515625, 6.60498046875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 100.79797601699829, "loadAverage": [ 9.57373046875, 6.9853515625, 6.60498046875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 102.90862202644348, "loadAverage": [ 9.57373046875, 6.9853515625, 6.60498046875 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 105.01938390731812, "loadAverage": [ 9.046875, 6.9189453125, 6.58349609375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 107.12016487121582, "loadAverage": [ 9.046875, 6.9189453125, 6.58349609375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 109.21345591545105, "loadAverage": [ 8.8828125, 6.919921875, 6.58544921875 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 111.30961894989014, "loadAverage": [ 8.8828125, 6.919921875, 6.58544921875 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 113.40875887870789, "loadAverage": [ 8.57177734375, 6.8876953125, 6.57568359375 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 115.50743103027344, "loadAverage": [ 8.57177734375, 6.8876953125, 6.57568359375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 117.60276007652283, "loadAverage": [ 8.57177734375, 6.8876953125, 6.57568359375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 119.69578790664673, "loadAverage": [ 8.04541015625, 6.80615234375, 6.548828125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 121.79111695289612, "loadAverage": [ 8.04541015625, 6.80615234375, 6.548828125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 123.88933205604553, "loadAverage": [ 7.64111328125, 6.74267578125, 6.52783203125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 125.99483299255371, "loadAverage": [ 7.64111328125, 6.74267578125, 6.52783203125 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 128.09067296981812, "loadAverage": [ 7.64111328125, 6.74267578125, 6.52783203125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 130.18932485580444, "loadAverage": [ 7.26904296875, 6.68017578125, 6.5068359375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 132.28633880615234, "loadAverage": [ 7.26904296875, 6.68017578125, 6.5068359375 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 134.3813338279724, "loadAverage": [ 6.7666015625, 6.58544921875, 6.47412109375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 136.48352003097534, "loadAverage": [ 6.7666015625, 6.58544921875, 6.47412109375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 138.5828959941864, "loadAverage": [ 6.224609375, 6.47607421875, 6.43603515625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 140.6810760498047, "loadAverage": [ 6.224609375, 6.47607421875, 6.43603515625 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 142.77963495254517, "loadAverage": [ 6.224609375, 6.47607421875, 6.43603515625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 144.87397694587708, "loadAverage": [ 5.88623046875, 6.4013671875, 6.40966796875 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 146.9753019809723, "loadAverage": [ 5.88623046875, 6.4013671875, 6.40966796875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 149.07011604309082, "loadAverage": [ 5.57470703125, 6.328125, 6.3837890625 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 151.16134595870972, "loadAverage": [ 5.57470703125, 6.328125, 6.3837890625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 153.25694799423218, "loadAverage": [ 5.57470703125, 6.328125, 6.3837890625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 155.35435509681702, "loadAverage": [ 5.4482421875, 6.2890625, 6.36962890625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 157.4474561214447, "loadAverage": [ 5.4482421875, 6.2890625, 6.36962890625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 159.54077196121216, "loadAverage": [ 5.251953125, 6.234375, 6.349609375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 161.6666510105133, "loadAverage": [ 5.251953125, 6.234375, 6.349609375 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15824715776, "processSeconds": 163.83803296089172, "profiled": false, "prompt": "Summarize this fictional incident report for a manager. Explain the timeline, user impact, cause, recovery, and three concrete follow-up actions. Distinguish confirmed facts from open questions. Use clear paragraphs and a short action list.\n\n09:02: The team deployed version 2.8 of the document search service. 09:07: Monitoring showed the 95th percentile search latency rising from 180 milliseconds to 4.2 seconds. 09:10: Support received six reports of slow searches; document uploads and downloads remained available. 09:13: The on-call engineer paused the rollout at 40 percent of traffic. 09:17: Logs showed repeated cache misses for queries containing a language filter. A new cache key included a randomly generated request identifier. 09:21: The engineer rolled traffic back to version 2.7. 09:26: Latency returned below 250 milliseconds. 09:32: The team confirmed no documents were lost or modified. Approximately 1,800 search requests experienced elevated latency; 73 timed out. The exact number of affected users is still being calculated. The release tests covered result correctness but did not assert cache reuse. A separate warning about database connection count appeared at 09:15; the team has not established whether it was a cause or an effect. Proposed follow-ups include a deterministic cache-key unit test, a staging workload with repeated filtered queries, and an automatic rollback threshold for latency.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624739.227365, "uncontended": false, "workload": "summary" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.9972201099688833 }, "endToEndSpeedups": { "adaptive": 1.7700554388315375 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.65985405445099, "decodeTokensPerSecond": 11.819100874661402, "elapsedSeconds": 25.363741993904114, "endToEndTokensPerSecond": 10.093147929888527, "expectedMTPActive": false, "firstTokenSeconds": 0.08659505844116211, "generatedTokens": 256, "loadSeconds": 2.205371856689453e-05, "name": "baseline", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "disabled", "prefillSeconds": 3.7024641036987305, "prefillTokensPerSecond": 91.02046382119947, "promptTokens": 337, "residentMemoryAfterBytes": 15414312960, "residentMemoryBeforeBytes": 15414116352, "ttftSeconds": 3.7890812158584595 }, { "acceleration": { "acceptanceRate": 0.5889967637540453, "acceptedDraftTokens": 182, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 309, "rounds": 73, "route": "mtp-speculative" }, "decodeSeconds": 10.845000982284546, "decodeTokensPerSecond": 23.60534594862457, "elapsedSeconds": 14.329348921775818, "endToEndTokensPerSecond": 17.865431388230462, "expectedMTPActive": true, "firstTokenSeconds": 0.00028705596923828125, "generatedTokens": 256, "loadSeconds": 2.002716064453125e-05, "name": "adaptive", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "adaptive", "prefillSeconds": 3.4826589822769165, "prefillTokensPerSecond": 96.76514459640657, "promptTokens": 337, "residentMemoryAfterBytes": 15824519168, "residentMemoryBeforeBytes": 15824388096, "ttftSeconds": 3.4829660654067993 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.969463776507501 }, "endToEndSpeedups": { "adaptive": 1.7276676598727985 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 21.422978043556213, "decodeTokensPerSecond": 11.949785855146404, "elapsedSeconds": 24.794650077819824, "endToEndTokensPerSecond": 10.32480794028249, "expectedMTPActive": false, "firstTokenSeconds": 0.10324704647064209, "generatedTokens": 256, "loadSeconds": 2.4080276489257812e-05, "name": "baseline", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "disabled", "prefillSeconds": 3.3696060180664062, "prefillTokensPerSecond": 100.01169222548516, "promptTokens": 337, "residentMemoryAfterBytes": 15414509568, "residentMemoryBeforeBytes": 15414427648, "ttftSeconds": 3.4728771448135376 }, { "acceleration": { "acceptanceRate": 0.5889967637540453, "acceptedDraftTokens": 182, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 309, "rounds": 73, "route": "mtp-speculative" }, "decodeSeconds": 10.87756896018982, "decodeTokensPerSecond": 23.534670378732553, "elapsedSeconds": 14.351516008377075, "endToEndTokensPerSecond": 17.837836772823938, "expectedMTPActive": true, "firstTokenSeconds": 0.00037097930908203125, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "adaptive", "prefillSeconds": 3.472319006919861, "prefillTokensPerSecond": 97.05329473714964, "promptTokens": 337, "residentMemoryAfterBytes": 15824617472, "residentMemoryBeforeBytes": 15824519168, "ttftSeconds": 3.472707986831665 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.0225575195070036 }, "endToEndSpeedups": { "adaptive": 1.7601250972052673 }, "greedyOutputParity": true, "promptCharacters": 1428, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 22.97916603088379, "decodeTokensPerSecond": 11.140526146855736, "elapsedSeconds": 26.31876504421234, "endToEndTokensPerSecond": 9.7269001630567, "expectedMTPActive": false, "firstTokenSeconds": 0.16310596466064453, "generatedTokens": 256, "loadSeconds": 0.00010597705841064453, "name": "baseline", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "disabled", "prefillSeconds": 3.332718014717102, "prefillTokensPerSecond": 101.11866605930243, "promptTokens": 337, "residentMemoryAfterBytes": 15414607872, "residentMemoryBeforeBytes": 15414509568, "ttftSeconds": 3.4959299564361572 }, { "acceleration": { "acceptanceRate": 0.5889967637540453, "acceptedDraftTokens": 182, "draftHistoryTokens": 336, "draftModel": "qwen-mtp", "draftedTokens": 309, "rounds": 73, "route": "mtp-speculative" }, "decodeSeconds": 11.361440062522888, "decodeTokensPerSecond": 22.532354929587456, "elapsedSeconds": 14.952780961990356, "endToEndTokensPerSecond": 17.120561095006103, "expectedMTPActive": true, "firstTokenSeconds": 0.0005140304565429688, "generatedTokens": 256, "loadSeconds": 2.300739288330078e-05, "name": "adaptive", "outputSHA256": "132da8ac8fd12147253a9a43470a0d7ece9db9db889d8ea4ccd40e1bf193e511", "policy": "adaptive", "prefillSeconds": 3.589138984680176, "prefillTokensPerSecond": 93.89438565584824, "promptTokens": 337, "residentMemoryAfterBytes": 15824715776, "residentMemoryBeforeBytes": 15824617472, "ttftSeconds": 3.589676022529602 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-reversed/ornith-code", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "7a363e5e29d3b5de5158e7c0fdec88a13a2e8b64988690a1962f07e1ca168c42", "receipt": { "after": { "gpuDeviceUtilizationPercent": 56, "loadAverage": [ 7.4853515625, 6.65234375, 6.49365234375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 41, "loadAverage": [ 5.3916015625, 6.2470703125, 6.35302734375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 46, "elapsedSeconds": 0.16013693809509277, "loadAverage": [ 5.3916015625, 6.2470703125, 6.35302734375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 2.2693281173706055, "loadAverage": [ 5.3916015625, 6.2470703125, 6.35302734375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 4.380108118057251, "loadAverage": [ 5.43994140625, 6.24267578125, 6.3505859375 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 6.880384922027588, "loadAverage": [ 5.43994140625, 6.24267578125, 6.3505859375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 9.110204219818115, "loadAverage": [ 5.43994140625, 6.24267578125, 6.3505859375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 11.383702993392944, "loadAverage": [ 5.484375, 6.23828125, 6.34814453125 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 13.67365026473999, "loadAverage": [ 5.484375, 6.23828125, 6.34814453125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 15.937946081161499, "loadAverage": [ 6.005859375, 6.33349609375, 6.380859375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 18.26474905014038, "loadAverage": [ 6.005859375, 6.33349609375, 6.380859375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 20.63537096977234, "loadAverage": [ 6.16552734375, 6.36083984375, 6.39013671875 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 22.894519090652466, "loadAverage": [ 6.16552734375, 6.36083984375, 6.39013671875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 25.011109113693237, "loadAverage": [ 6.15185546875, 6.3544921875, 6.3876953125 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 27.13076400756836, "loadAverage": [ 6.15185546875, 6.3544921875, 6.3876953125 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 29.56958508491516, "loadAverage": [ 7.1806640625, 6.56396484375, 6.46142578125 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 31.767003059387207, "loadAverage": [ 7.1806640625, 6.56396484375, 6.46142578125 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 33.94800901412964, "loadAverage": [ 7.1806640625, 6.56396484375, 6.46142578125 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 36.37644600868225, "loadAverage": [ 7.0859375, 6.55419921875, 6.45849609375 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 38.58680486679077, "loadAverage": [ 7.0859375, 6.55419921875, 6.45849609375 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 40.85073804855347, "loadAverage": [ 6.91845703125, 6.5283203125, 6.44970703125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 43.1207320690155, "loadAverage": [ 6.91845703125, 6.5283203125, 6.44970703125 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 45.34003925323486, "loadAverage": [ 7.4853515625, 6.65234375, 6.49365234375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9464430592, "processSeconds": 47.43175292015076, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624903.449096, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9697399745616944 }, "endToEndSpeedups": { "adaptive": 0.9696557702055493 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 4.8227680921554565, "decodeTokensPerSecond": 53.08154883424739, "elapsedSeconds": 5.072595000267029, "endToEndTokensPerSecond": 50.46726576565324, "expectedMTPActive": true, "firstTokenSeconds": 0.0014580488204956055, "generatedTokens": 256, "loadSeconds": 2.7060508728027344e-05, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.24440503120422363, "prefillTokensPerSecond": 220.9447151473648, "promptTokens": 54, "residentMemoryAfterBytes": 9463283712, "residentMemoryBeforeBytes": 9463054336, "ttftSeconds": 0.24589014053344727 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 4.676831007003784, "decodeTokensPerSecond": 54.737919676085674, "elapsedSeconds": 4.918671011924744, "endToEndTokensPerSecond": 52.046579122563365, "expectedMTPActive": false, "firstTokenSeconds": 0.043910980224609375, "generatedTokens": 256, "loadSeconds": 1.5974044799804688e-05, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.2331550121307373, "prefillTokensPerSecond": 231.60557221785356, "promptTokens": 54, "residentMemoryAfterBytes": 9334571008, "residentMemoryBeforeBytes": 9334554624, "ttftSeconds": 0.2770819664001465 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9509105511634649 }, "endToEndSpeedups": { "adaptive": 0.9494191091210825 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 5.3581401109695435, "decodeTokensPerSecond": 47.77777264090195, "elapsedSeconds": 5.470210075378418, "endToEndTokensPerSecond": 46.79893394812455, "expectedMTPActive": true, "firstTokenSeconds": 0.0004889965057373047, "generatedTokens": 256, "loadSeconds": 1.3113021850585938e-05, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.10776698589324951, "prefillTokensPerSecond": 501.08110153039496, "promptTokens": 54, "residentMemoryAfterBytes": 9463529472, "residentMemoryBeforeBytes": 9463382016, "ttftSeconds": 0.1082690954208374 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 5.095111966133118, "decodeTokensPerSecond": 50.24423441557626, "elapsedSeconds": 5.193521976470947, "endToEndTokensPerSecond": 49.29217613014794, "expectedMTPActive": false, "firstTokenSeconds": 0.01553201675415039, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.09571802616119385, "prefillTokensPerSecond": 564.1570576169357, "promptTokens": 54, "residentMemoryAfterBytes": 9334620160, "residentMemoryBeforeBytes": 9334571008, "ttftSeconds": 0.11126303672790527 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.964649261806438 }, "endToEndSpeedups": { "adaptive": 0.9639872399683244 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.5186721991701245, "acceptedDraftTokens": 125, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 241, "rounds": 131, "route": "mtp-speculative" }, "decodeSeconds": 5.836509943008423, "decodeTokensPerSecond": 43.861828815465884, "elapsedSeconds": 5.948752999305725, "endToEndTokensPerSecond": 43.034229195577225, "expectedMTPActive": true, "firstTokenSeconds": 0.0003819465637207031, "generatedTokens": 256, "loadSeconds": 1.3947486877441406e-05, "name": "adaptive", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "adaptive", "prefillSeconds": 0.10719597339630127, "prefillTokensPerSecond": 503.75026495037395, "promptTokens": 54, "residentMemoryAfterBytes": 9464315904, "residentMemoryBeforeBytes": 9463529472, "ttftSeconds": 0.10759186744689941 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 5.630185008049011, "decodeTokensPerSecond": 45.469198549251566, "elapsedSeconds": 5.734521985054016, "endToEndTokensPerSecond": 44.64190749764622, "expectedMTPActive": false, "firstTokenSeconds": 0.01823902130126953, "generatedTokens": 256, "loadSeconds": 1.3947486877441406e-05, "name": "baseline", "outputSHA256": "cd134369dacefc1cc1bc8bceceb5cfb2dc106e35c077361b20e79108d05fabb4", "policy": "disabled", "prefillSeconds": 0.10044896602630615, "prefillTokensPerSecond": 537.586419613898, "promptTokens": 54, "residentMemoryAfterBytes": 9334636544, "residentMemoryBeforeBytes": 9334620160, "ttftSeconds": 0.11870193481445312 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-reversed/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "205ea121e5356a8f58e18a2f65a65a573212390cd74b08861ef50860ac8ae1f0", "receipt": { "after": { "gpuDeviceUtilizationPercent": 58, "loadAverage": [ 9.16796875, 7.2255859375, 6.71240234375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 55, "loadAverage": [ 7.4853515625, 6.65234375, 6.49365234375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 57, "elapsedSeconds": 0.09006428718566895, "loadAverage": [ 7.4853515625, 6.65234375, 6.49365234375 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 2.1915950775146484, "loadAverage": [ 7.2861328125, 6.62451171875, 6.48486328125 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 4.300493240356445, "loadAverage": [ 7.2861328125, 6.62451171875, 6.48486328125 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 6.476424217224121, "loadAverage": [ 7.2861328125, 6.62451171875, 6.48486328125 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 8.656381368637085, "loadAverage": [ 7.10302734375, 6.59716796875, 6.47607421875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 10.838325023651123, "loadAverage": [ 7.10302734375, 6.59716796875, 6.47607421875 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 13.098226308822632, "loadAverage": [ 7.0947265625, 6.603515625, 6.47900390625 ] }, { "deviceUtilizationPercent": 52, "elapsedSeconds": 15.462062120437622, "loadAverage": [ 7.0947265625, 6.603515625, 6.47900390625 ] }, { "deviceUtilizationPercent": 45, "elapsedSeconds": 17.661643028259277, "loadAverage": [ 8.8486328125, 6.97509765625, 6.61083984375 ] }, { "deviceUtilizationPercent": 51, "elapsedSeconds": 19.870218992233276, "loadAverage": [ 8.8486328125, 6.97509765625, 6.61083984375 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 22.082136154174805, "loadAverage": [ 10.1416015625, 7.27392578125, 6.71826171875 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 24.289380073547363, "loadAverage": [ 10.1416015625, 7.27392578125, 6.71826171875 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 26.425503253936768, "loadAverage": [ 10.1416015625, 7.27392578125, 6.71826171875 ] }, { "deviceUtilizationPercent": 77, "elapsedSeconds": 28.54460120201111, "loadAverage": [ 9.8095703125, 7.25244140625, 6.7138671875 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 30.671943187713623, "loadAverage": [ 9.8095703125, 7.25244140625, 6.7138671875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 32.81610131263733, "loadAverage": [ 9.18408203125, 7.1650390625, 6.68603515625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 34.93916320800781, "loadAverage": [ 9.18408203125, 7.1650390625, 6.68603515625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 37.10272932052612, "loadAverage": [ 9.0087890625, 7.162109375, 6.6875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 39.2621853351593, "loadAverage": [ 9.0087890625, 7.162109375, 6.6875 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 41.42637634277344, "loadAverage": [ 9.0087890625, 7.162109375, 6.6875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 43.607195138931274, "loadAverage": [ 9.16796875, 7.2255859375, 6.71240234375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9465462784, "processSeconds": 45.714460134506226, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624951.0699048, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.7290370582813314 }, "endToEndSpeedups": { "adaptive": 0.7245269797054104 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 4.183992981910706, "decodeTokensPerSecond": 61.185571081692494, "elapsedSeconds": 4.51315701007843, "endToEndTokensPerSecond": 56.72304318868605, "expectedMTPActive": true, "firstTokenSeconds": 0.0004290342330932617, "generatedTokens": 256, "loadSeconds": 1.9073486328125e-05, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.32599902153015137, "prefillTokensPerSecond": 184.0496321687599, "promptTokens": 60, "residentMemoryAfterBytes": 9463300096, "residentMemoryBeforeBytes": 9463218176, "ttftSeconds": 0.32644712924957275 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.0502859354019165, "decodeTokensPerSecond": 83.9265581724123, "elapsedSeconds": 3.2699040174484253, "endToEndTokensPerSecond": 78.28975977091895, "expectedMTPActive": false, "firstTokenSeconds": 0.011642932891845703, "generatedTokens": 256, "loadSeconds": 1.0013580322265625e-05, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.21694302558898926, "prefillTokensPerSecond": 276.5703107398962, "promptTokens": 60, "residentMemoryAfterBytes": 9351397376, "residentMemoryBeforeBytes": 9351364608, "ttftSeconds": 0.22859597206115723 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.3269960939039667 }, "endToEndSpeedups": { "adaptive": 0.3500277158650747 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 9.989682078361511, "decodeTokensPerSecond": 25.626441161177436, "elapsedSeconds": 10.10204291343689, "endToEndTokensPerSecond": 25.34140888072157, "expectedMTPActive": true, "firstTokenSeconds": 0.0004950761795043945, "generatedTokens": 256, "loadSeconds": 1.2040138244628906e-05, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.108254075050354, "prefillTokensPerSecond": 554.2516526245429, "promptTokens": 60, "residentMemoryAfterBytes": 9465380864, "residentMemoryBeforeBytes": 9463398400, "ttftSeconds": 0.10876119136810303 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.266587018966675, "decodeTokensPerSecond": 78.36925773401896, "elapsedSeconds": 3.5359950065612793, "endToEndTokensPerSecond": 72.39829228405996, "expectedMTPActive": false, "firstTokenSeconds": 0.016772985458374023, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.26670897006988525, "prefillTokensPerSecond": 224.9643121649726, "promptTokens": 60, "residentMemoryAfterBytes": 9351430144, "residentMemoryBeforeBytes": 9351397376, "ttftSeconds": 0.28349292278289795 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.621524216585805 }, "endToEndSpeedups": { "adaptive": 0.6275601397565709 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.25892857142857145, "acceptedDraftTokens": 29, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 112, "rounds": 69, "route": "mtp-speculative" }, "decodeSeconds": 5.7626200914382935, "decodeTokensPerSecond": 44.42423688147468, "elapsedSeconds": 6.014245986938477, "endToEndTokensPerSecond": 42.56560183204538, "expectedMTPActive": true, "firstTokenSeconds": 0.0011860132217407227, "generatedTokens": 256, "loadSeconds": 1.5020370483398438e-05, "name": "adaptive", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "adaptive", "prefillSeconds": 0.2456960678100586, "prefillTokensPerSecond": 244.20415245059795, "promptTokens": 60, "residentMemoryAfterBytes": 9465479168, "residentMemoryBeforeBytes": 9465380864, "ttftSeconds": 0.24689710140228271 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.581607937812805, "decodeTokensPerSecond": 71.4762767016684, "elapsedSeconds": 3.774301052093506, "endToEndTokensPerSecond": 67.82712784874526, "expectedMTPActive": false, "firstTokenSeconds": 0.01258695125579834, "generatedTokens": 256, "loadSeconds": 1.0967254638671875e-05, "name": "baseline", "outputSHA256": "ca3ac87c453ea9cb8fdd5fe572ce7e0e9930e17487f37393e45606052e4db11e", "policy": "disabled", "prefillSeconds": 0.1898289918899536, "prefillTokensPerSecond": 316.0739537340155, "promptTokens": 60, "residentMemoryAfterBytes": 9351430144, "residentMemoryBeforeBytes": 9351430144, "ttftSeconds": 0.20242691040039062 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-reversed/qwen-code", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "abb92db3c11dfc7b9ef212bd2a6f82323e5d7ba4aeefe4512ef315db424f0d8a", "receipt": { "after": { "gpuDeviceUtilizationPercent": 56, "loadAverage": [ 24.96533203125, 13.49462890625, 9.2548828125 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 52, "loadAverage": [ 9.16796875, 7.2255859375, 6.71240234375 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 54, "elapsedSeconds": 0.09082221984863281, "loadAverage": [ 9.16796875, 7.2255859375, 6.71240234375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 2.187695026397705, "loadAverage": [ 8.75390625, 7.171875, 6.6962890625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 4.282471179962158, "loadAverage": [ 8.75390625, 7.171875, 6.6962890625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 6.375388145446777, "loadAverage": [ 8.373046875, 7.119140625, 6.68017578125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 8.471826076507568, "loadAverage": [ 8.373046875, 7.119140625, 6.68017578125 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 10.568481922149658, "loadAverage": [ 8.373046875, 7.119140625, 6.68017578125 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 12.666258096694946, "loadAverage": [ 7.9423828125, 7.05029296875, 6.658203125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 14.761430025100708, "loadAverage": [ 7.9423828125, 7.05029296875, 6.658203125 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 16.857935190200806, "loadAverage": [ 7.70654296875, 7.01611328125, 6.6484375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 18.955432176589966, "loadAverage": [ 7.70654296875, 7.01611328125, 6.6484375 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 21.05484700202942, "loadAverage": [ 7.48974609375, 6.982421875, 6.638671875 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 23.15466809272766, "loadAverage": [ 7.48974609375, 6.982421875, 6.638671875 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 25.271913290023804, "loadAverage": [ 7.48974609375, 6.982421875, 6.638671875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 28.085982084274292, "loadAverage": [ 7.2099609375, 6.9326171875, 6.623046875 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 31.07688808441162, "loadAverage": [ 8.55419921875, 7.2158203125, 6.724609375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 33.949666023254395, "loadAverage": [ 8.55419921875, 7.2158203125, 6.724609375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 36.49758005142212, "loadAverage": [ 9.310546875, 7.39453125, 6.79052734375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 39.03070521354675, "loadAverage": [ 9.310546875, 7.39453125, 6.79052734375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 41.936532974243164, "loadAverage": [ 9.92626953125, 7.5537109375, 6.85009765625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 44.47289705276489, "loadAverage": [ 9.92626953125, 7.5537109375, 6.85009765625 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 46.673907995224, "loadAverage": [ 10.09228515625, 7.62744140625, 6.8798828125 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 48.845974922180176, "loadAverage": [ 10.09228515625, 7.62744140625, 6.8798828125 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 51.771377086639404, "loadAverage": [ 14.48876953125, 8.57958984375, 7.22021484375 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 55.16762590408325, "loadAverage": [ 14.48876953125, 8.57958984375, 7.22021484375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 58.616517066955566, "loadAverage": [ 16.69140625, 9.13427734375, 7.423828125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 61.96666693687439, "loadAverage": [ 17.5966796875, 9.447265625, 7.5439453125 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 64.98756289482117, "loadAverage": [ 17.5966796875, 9.447265625, 7.5439453125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 68.51158595085144, "loadAverage": [ 18.029296875, 9.671875, 7.63427734375 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 71.71408987045288, "loadAverage": [ 18.02685546875, 9.81005859375, 7.69482421875 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 75.1402440071106, "loadAverage": [ 18.02685546875, 9.81005859375, 7.69482421875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 78.40740299224854, "loadAverage": [ 18.3447265625, 10.01220703125, 7.7783203125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 81.12364506721497, "loadAverage": [ 17.83642578125, 10.044921875, 7.802734375 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 84.23508501052856, "loadAverage": [ 17.83642578125, 10.044921875, 7.802734375 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 87.3729362487793, "loadAverage": [ 18.81005859375, 10.3759765625, 7.9326171875 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 90.80643916130066, "loadAverage": [ 19.3056640625, 10.61865234375, 8.0322265625 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 94.0080680847168, "loadAverage": [ 19.3056640625, 10.61865234375, 8.0322265625 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 96.96449518203735, "loadAverage": [ 18.80029296875, 10.65771484375, 8.06103515625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 100.20592999458313, "loadAverage": [ 18.80029296875, 10.65771484375, 8.06103515625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 103.37519717216492, "loadAverage": [ 20.81787109375, 11.2109375, 8.271484375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 106.22744917869568, "loadAverage": [ 21.712890625, 11.5556640625, 8.41015625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 109.31918215751648, "loadAverage": [ 21.712890625, 11.5556640625, 8.41015625 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 112.36410212516785, "loadAverage": [ 22.21630859375, 11.82861328125, 8.52490234375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 115.42504000663757, "loadAverage": [ 22.21630859375, 11.82861328125, 8.52490234375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 118.15333819389343, "loadAverage": [ 22.99951171875, 12.1630859375, 8.662109375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 120.84269094467163, "loadAverage": [ 22.59912109375, 12.259765625, 8.716796875 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 123.56349301338196, "loadAverage": [ 22.59912109375, 12.259765625, 8.716796875 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 126.41029405593872, "loadAverage": [ 22.310546875, 12.37158203125, 8.77685546875 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 128.8889079093933, "loadAverage": [ 22.310546875, 12.37158203125, 8.77685546875 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 131.7235550880432, "loadAverage": [ 21.16455078125, 12.298828125, 8.77197265625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 134.71923208236694, "loadAverage": [ 21.16455078125, 12.298828125, 8.77197265625 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 137.55205392837524, "loadAverage": [ 20.1904296875, 12.24365234375, 8.77294921875 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 140.4294581413269, "loadAverage": [ 20.1904296875, 12.24365234375, 8.77294921875 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 142.81683921813965, "loadAverage": [ 20.1748046875, 12.3720703125, 8.83837890625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 145.87299609184265, "loadAverage": [ 24.96533203125, 13.49462890625, 9.2548828125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 148.3839132785797, "loadAverage": [ 24.96533203125, 13.49462890625, 9.2548828125 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15827419136, "processSeconds": 150.48906326293945, "profiled": false, "prompt": "Write a complete Python implementation of an LRU cache using a dictionary and a doubly linked list, with get and put operations, type hints, and a short usage example. Return the code directly without introductory prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788624996.980743, "uncontended": false, "workload": "code" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.7062322161950907 }, "endToEndSpeedups": { "adaptive": 2.564296334232043 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 10.30733609199524, "decodeTokensPerSecond": 24.836679207425057, "elapsedSeconds": 11.090577006340027, "endToEndTokensPerSecond": 23.082658355255578, "expectedMTPActive": true, "firstTokenSeconds": 0.00039505958557128906, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.7816139459609985, "prefillTokensPerSecond": 69.08781538385514, "promptTokens": 54, "residentMemoryAfterBytes": 15823011840, "residentMemoryBeforeBytes": 15822815232, "ttftSeconds": 0.782027006149292 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 27.894044995307922, "decodeTokensPerSecond": 9.177586113561585, "elapsedSeconds": 28.439525961875916, "endToEndTokensPerSecond": 9.001556507769367, "expectedMTPActive": false, "firstTokenSeconds": 0.08211696147918701, "generatedTokens": 256, "loadSeconds": 6.604194641113281e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.5338560342788696, "prefillTokensPerSecond": 101.1508656504051, "promptTokens": 54, "residentMemoryAfterBytes": 15438200832, "residentMemoryBeforeBytes": 15438020608, "ttftSeconds": 0.6160390377044678 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.829251187701691 }, "endToEndSpeedups": { "adaptive": 1.7434668121087449 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 11.163246035575867, "decodeTokensPerSecond": 22.932397905068118, "elapsedSeconds": 11.991575002670288, "endToEndTokensPerSecond": 21.348321629393457, "expectedMTPActive": true, "firstTokenSeconds": 0.0006200075149536133, "generatedTokens": 256, "loadSeconds": 2.396106719970703e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.8256739377975464, "prefillTokensPerSecond": 65.40111965268389, "promptTokens": 54, "residentMemoryAfterBytes": 15825649664, "residentMemoryBeforeBytes": 15823110144, "ttftSeconds": 0.8263179063796997 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 20.42038106918335, "decodeTokensPerSecond": 12.536494746727953, "elapsedSeconds": 20.90691304206848, "endToEndTokensPerSecond": 12.24475366042236, "expectedMTPActive": false, "firstTokenSeconds": 0.13705599308013916, "generatedTokens": 256, "loadSeconds": 4.8995018005371094e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.47617292404174805, "prefillTokensPerSecond": 113.40418002277173, "promptTokens": 54, "residentMemoryAfterBytes": 15438200832, "residentMemoryBeforeBytes": 15438200832, "ttftSeconds": 0.6132779121398926 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.6927014061685024 }, "endToEndSpeedups": { "adaptive": 1.6781877368114007 }, "greedyOutputParity": true, "promptCharacters": 220, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.7236363636363636, "acceptedDraftTokens": 199, "draftHistoryTokens": 53, "draftModel": "qwen-mtp", "draftedTokens": 275, "rounds": 57, "route": "mtp-speculative" }, "decodeSeconds": 10.624022006988525, "decodeTokensPerSecond": 24.096335628032598, "elapsedSeconds": 11.259902954101562, "endToEndTokensPerSecond": 22.73554230827085, "expectedMTPActive": true, "firstTokenSeconds": 0.001961946487426758, "generatedTokens": 256, "loadSeconds": 4.8995018005371094e-05, "name": "adaptive", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "adaptive", "prefillSeconds": 0.610366940498352, "prefillTokensPerSecond": 88.47137093616196, "promptTokens": 54, "residentMemoryAfterBytes": 15827419136, "residentMemoryBeforeBytes": 15825666048, "ttftSeconds": 0.6123778820037842 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 17.983296990394592, "decodeTokensPerSecond": 14.23543191978296, "elapsedSeconds": 18.896231055259705, "endToEndTokensPerSecond": 13.547675155503734, "expectedMTPActive": false, "firstTokenSeconds": 0.05465507507324219, "generatedTokens": 256, "loadSeconds": 6.496906280517578e-05, "name": "baseline", "outputSHA256": "ca2c0ddf75b621ec62d587ff04b6f89f528aafe8d744b0730c298166deb5f461", "policy": "disabled", "prefillSeconds": 0.9047880172729492, "prefillTokensPerSecond": 59.68248801830641, "promptTokens": 54, "residentMemoryAfterBytes": 15438233600, "residentMemoryBeforeBytes": 15438200832, "ttftSeconds": 0.9595080614089966 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-reversed/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "71622d0f9f1246a5daae8cc7eca62881adf88fe866319e8fe6fde6338fc2635b", "receipt": { "after": { "gpuDeviceUtilizationPercent": 69, "loadAverage": [ 45.1962890625, 25.31640625, 14.9296875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 37, "loadAverage": [ 23.3662109375, 13.353515625, 9.2294921875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0", "--top-p", "1", "--json", "--variants", "adaptive,baseline", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 34, "elapsedSeconds": 0.09883594512939453, "loadAverage": [ 23.3662109375, 13.353515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 2.203329086303711, "loadAverage": [ 23.3662109375, 13.353515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 4.335525274276733, "loadAverage": [ 23.3662109375, 13.353515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 6.521083116531372, "loadAverage": [ 21.89501953125, 13.21435546875, 9.20458984375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 8.710731029510498, "loadAverage": [ 21.89501953125, 13.21435546875, 9.20458984375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 10.94186019897461, "loadAverage": [ 20.5419921875, 13.07763671875, 9.1796875 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 13.139568090438843, "loadAverage": [ 20.5419921875, 13.07763671875, 9.1796875 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 15.344344139099121, "loadAverage": [ 19.69775390625, 13.0263671875, 9.18408203125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 17.541656017303467, "loadAverage": [ 19.69775390625, 13.0263671875, 9.18408203125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 19.913045167922974, "loadAverage": [ 19.4814453125, 13.09228515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 22.21728205680847, "loadAverage": [ 19.4814453125, 13.09228515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 81, "elapsedSeconds": 24.702322244644165, "loadAverage": [ 19.4814453125, 13.09228515625, 9.2294921875 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 27.480812072753906, "loadAverage": [ 18.72216796875, 13.04052734375, 9.23388671875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 30.117607355117798, "loadAverage": [ 20.265625, 13.45458984375, 9.40234375 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 32.73131322860718, "loadAverage": [ 20.265625, 13.45458984375, 9.40234375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 35.086735248565674, "loadAverage": [ 22.88671875, 14.11083984375, 9.65771484375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 37.52501034736633, "loadAverage": [ 22.88671875, 14.11083984375, 9.65771484375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 39.94181513786316, "loadAverage": [ 26.3388671875, 14.97216796875, 9.98779296875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 42.311331033706665, "loadAverage": [ 26.3388671875, 14.97216796875, 9.98779296875 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 44.7534441947937, "loadAverage": [ 26.3388671875, 14.97216796875, 9.98779296875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 47.112189292907715, "loadAverage": [ 30.1552734375, 15.95166015625, 10.36279296875 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 49.54869723320007, "loadAverage": [ 30.1552734375, 15.95166015625, 10.36279296875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 52.02789306640625, "loadAverage": [ 33.666015625, 16.9150390625, 10.7353515625 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 54.4402060508728, "loadAverage": [ 33.666015625, 16.9150390625, 10.7353515625 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 56.98220705986023, "loadAverage": [ 38.2568359375, 18.14453125, 11.20556640625 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 59.370721101760864, "loadAverage": [ 38.2568359375, 18.14453125, 11.20556640625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 62.32659411430359, "loadAverage": [ 36.39404296875, 18.09228515625, 11.2275390625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 64.5657651424408, "loadAverage": [ 36.39404296875, 18.09228515625, 11.2275390625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 66.91111922264099, "loadAverage": [ 34.4404296875, 17.99072265625, 11.23193359375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 69.82606625556946, "loadAverage": [ 34.4404296875, 17.99072265625, 11.23193359375 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 72.70674300193787, "loadAverage": [ 32.64306640625, 17.89111328125, 11.236328125 ] }, { "deviceUtilizationPercent": 44, "elapsedSeconds": 74.94132828712463, "loadAverage": [ 32.03076171875, 18.0087890625, 11.31689453125 ] }, { "deviceUtilizationPercent": 69, "elapsedSeconds": 77.11666321754456, "loadAverage": [ 32.03076171875, 18.0087890625, 11.31689453125 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 80.20499300956726, "loadAverage": [ 32.50830078125, 18.34033203125, 11.47314453125 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 83.24004411697388, "loadAverage": [ 32.50830078125, 18.34033203125, 11.47314453125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 86.16507601737976, "loadAverage": [ 31.34619140625, 18.33447265625, 11.51123046875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 89.49692702293396, "loadAverage": [ 31.34619140625, 18.33447265625, 11.51123046875 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 92.49102115631104, "loadAverage": [ 31.318359375, 18.54443359375, 11.625 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 95.74700808525085, "loadAverage": [ 30.73193359375, 18.634765625, 11.697265625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 98.98880004882812, "loadAverage": [ 30.73193359375, 18.634765625, 11.697265625 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 102.21768116950989, "loadAverage": [ 30.6728515625, 18.8232421875, 11.80419921875 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 105.45421528816223, "loadAverage": [ 30.93896484375, 19.0751953125, 11.93408203125 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 108.9528341293335, "loadAverage": [ 30.93896484375, 19.0751953125, 11.93408203125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 113.08783006668091, "loadAverage": [ 31.02392578125, 19.28955078125, 12.05126953125 ] }, { "deviceUtilizationPercent": 72, "elapsedSeconds": 116.20992612838745, "loadAverage": [ 29.26025390625, 19.11865234375, 12.033203125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 119.49017119407654, "loadAverage": [ 29.26025390625, 19.11865234375, 12.033203125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 122.69335603713989, "loadAverage": [ 30.6806640625, 19.5810546875, 12.23779296875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 125.75804805755615, "loadAverage": [ 30.3857421875, 19.7041015625, 12.32421875 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 128.8421173095703, "loadAverage": [ 30.3857421875, 19.7041015625, 12.32421875 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 132.4070770740509, "loadAverage": [ 31.5556640625, 20.1240234375, 12.515625 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 135.78558111190796, "loadAverage": [ 32.3115234375, 20.47021484375, 12.68212890625 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 139.08193016052246, "loadAverage": [ 32.3115234375, 20.47021484375, 12.68212890625 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 142.30391025543213, "loadAverage": [ 32.6064453125, 20.72802734375, 12.818359375 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 145.63004517555237, "loadAverage": [ 32.7978515625, 20.96484375, 12.9482421875 ] }, { "deviceUtilizationPercent": 66, "elapsedSeconds": 148.7469711303711, "loadAverage": [ 32.7978515625, 20.96484375, 12.9482421875 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 151.5499472618103, "loadAverage": [ 33.85498046875, 21.38037109375, 13.1416015625 ] }, { "deviceUtilizationPercent": 68, "elapsedSeconds": 154.47599911689758, "loadAverage": [ 33.85498046875, 21.38037109375, 13.1416015625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 157.22787714004517, "loadAverage": [ 33.9462890625, 21.6064453125, 13.26953125 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 160.0602991580963, "loadAverage": [ 33.5498046875, 21.72900390625, 13.361328125 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 162.94475317001343, "loadAverage": [ 33.5498046875, 21.72900390625, 13.361328125 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 166.2506172657013, "loadAverage": [ 33.02490234375, 21.81640625, 13.44091796875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 169.33834218978882, "loadAverage": [ 33.02490234375, 21.81640625, 13.44091796875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 172.93372702598572, "loadAverage": [ 32.6220703125, 21.9189453125, 13.52587890625 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 176.97347927093506, "loadAverage": [ 32.73193359375, 22.119140625, 13.6455078125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 180.91829299926758, "loadAverage": [ 32.3525390625, 22.21630859375, 13.7294921875 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 184.44381999969482, "loadAverage": [ 32.3525390625, 22.21630859375, 13.7294921875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 187.68330812454224, "loadAverage": [ 40.251953125, 24.02197265625, 14.41650390625 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 191.89226913452148, "loadAverage": [ 45.1962890625, 25.31640625, 14.9296875 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15810560000, "processSeconds": 194.2303249835968, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788625147.6817238, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.9923312310201227 }, "endToEndSpeedups": { "adaptive": 1.9345236327992004 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 14.611358046531677, "decodeTokensPerSecond": 17.52061643994599, "elapsedSeconds": 15.333928942680359, "endToEndTokensPerSecond": 16.69500367172377, "expectedMTPActive": true, "firstTokenSeconds": 0.0005730390548706055, "generatedTokens": 256, "loadSeconds": 4.601478576660156e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.7161870002746582, "prefillTokensPerSecond": 83.77700234294947, "promptTokens": 60, "residentMemoryAfterBytes": 15808299008, "residentMemoryBeforeBytes": 15805693952, "ttftSeconds": 0.7168060541152954 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 29.11066496372223, "decodeTokensPerSecond": 8.794027904172843, "elapsedSeconds": 29.66384792327881, "endToEndTokensPerSecond": 8.630033455609214, "expectedMTPActive": false, "firstTokenSeconds": 0.14921605587005615, "generatedTokens": 256, "loadSeconds": 0.00011408329010009766, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.5307199954986572, "prefillTokensPerSecond": 113.05396538456182, "promptTokens": 60, "residentMemoryAfterBytes": 15420030976, "residentMemoryBeforeBytes": 15419932672, "ttftSeconds": 0.6800501346588135 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.040428722509336 }, "endToEndSpeedups": { "adaptive": 1.0399402427823943 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 25.984102964401245, "decodeTokensPerSecond": 9.852177708452174, "elapsedSeconds": 26.39515507221222, "endToEndTokensPerSecond": 9.698749611420421, "expectedMTPActive": true, "firstTokenSeconds": 0.0010169744491577148, "generatedTokens": 256, "loadSeconds": 4.8995018005371094e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.4018000364303589, "prefillTokensPerSecond": 149.32801035322794, "promptTokens": 60, "residentMemoryAfterBytes": 15810428928, "residentMemoryBeforeBytes": 15808397312, "ttftSeconds": 0.402866005897522 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 27.03460705280304, "decodeTokensPerSecond": 9.469344218689395, "elapsedSeconds": 27.449383974075317, "endToEndTokensPerSecond": 9.32625665631623, "expectedMTPActive": false, "firstTokenSeconds": 0.23631703853607178, "generatedTokens": 256, "loadSeconds": 0.0005550384521484375, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.4052619934082031, "prefillTokensPerSecond": 148.05237346687125, "promptTokens": 60, "residentMemoryAfterBytes": 15420063744, "residentMemoryBeforeBytes": 15420047360, "ttftSeconds": 0.6421340703964233 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.9230810988761795 }, "endToEndSpeedups": { "adaptive": 1.9213973760606717 }, "greedyOutputParity": true, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0, "variants": [ { "acceleration": { "acceptanceRate": 0.4074074074074074, "acceptedDraftTokens": 132, "draftHistoryTokens": 59, "draftModel": "qwen-mtp", "draftedTokens": 324, "rounds": 124, "route": "mtp-speculative" }, "decodeSeconds": 15.953748941421509, "decodeTokensPerSecond": 16.046385143703404, "elapsedSeconds": 16.352140069007874, "endToEndTokensPerSecond": 15.655443197015874, "expectedMTPActive": true, "firstTokenSeconds": 0.0006709098815917969, "generatedTokens": 256, "loadSeconds": 5.698204040527344e-05, "name": "adaptive", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "adaptive", "prefillSeconds": 0.3868929147720337, "prefillTokensPerSecond": 155.08167172136868, "promptTokens": 60, "residentMemoryAfterBytes": 15810527232, "residentMemoryBeforeBytes": 15810428928, "ttftSeconds": 0.38762080669403076 }, { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 30.680353045463562, "decodeTokensPerSecond": 8.344102156212069, "elapsedSeconds": 31.4189590215683, "endToEndTokensPerSecond": 8.147946589327248, "expectedMTPActive": false, "firstTokenSeconds": 0.10634195804595947, "generatedTokens": 256, "loadSeconds": 5.3048133850097656e-05, "name": "baseline", "outputSHA256": "f0eee860afeee246903ecb69993c7b7fee39cc16b60c5bf6535276f31f58bcda", "policy": "disabled", "prefillSeconds": 0.7313799858093262, "prefillTokensPerSecond": 82.03669934118521, "promptTokens": 60, "residentMemoryAfterBytes": 15420112896, "residentMemoryBeforeBytes": 15420063744, "ttftSeconds": 0.8377749919891357 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-sampled/ornith-chat", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "51867457047c74db52cc1a60abb0e8d28c3d6a013dddd439b08cbe2dafaf3975", "receipt": { "after": { "gpuDeviceUtilizationPercent": 55, "loadAverage": [ 31.673828125, 25.80517578125, 15.90966796875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "before": { "gpuDeviceUtilizationPercent": 38, "loadAverage": [ 45.1962890625, 25.31640625, 14.9296875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0.7", "--top-p", "0.9", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 37, "elapsedSeconds": 0.23145008087158203, "loadAverage": [ 42.69775390625, 25.12841796875, 14.923828125 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 2.6275441646575928, "loadAverage": [ 42.69775390625, 25.12841796875, 14.923828125 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 4.822941064834595, "loadAverage": [ 42.69775390625, 25.12841796875, 14.923828125 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 7.084103107452393, "loadAverage": [ 41.3603515625, 25.142578125, 14.98828125 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 9.28956913948059, "loadAverage": [ 41.3603515625, 25.142578125, 14.98828125 ] }, { "deviceUtilizationPercent": 64, "elapsedSeconds": 11.569926977157593, "loadAverage": [ 40.1298828125, 25.15673828125, 15.052734375 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 13.851749897003174, "loadAverage": [ 40.1298828125, 25.15673828125, 15.052734375 ] }, { "deviceUtilizationPercent": 56, "elapsedSeconds": 16.082265853881836, "loadAverage": [ 43.802734375, 26.16650390625, 15.46826171875 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 18.473559141159058, "loadAverage": [ 43.802734375, 26.16650390625, 15.46826171875 ] }, { "deviceUtilizationPercent": 53, "elapsedSeconds": 21.090827226638794, "loadAverage": [ 45.66015625, 26.84423828125, 15.77001953125 ] }, { "deviceUtilizationPercent": 48, "elapsedSeconds": 23.562591075897217, "loadAverage": [ 45.66015625, 26.84423828125, 15.77001953125 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 26.099173069000244, "loadAverage": [ 46.32763671875, 27.294921875, 15.99365234375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 28.355140209197998, "loadAverage": [ 46.32763671875, 27.294921875, 15.99365234375 ] }, { "deviceUtilizationPercent": 60, "elapsedSeconds": 30.595892190933228, "loadAverage": [ 43.57861328125, 27.04052734375, 15.97021484375 ] }, { "deviceUtilizationPercent": 58, "elapsedSeconds": 32.94844603538513, "loadAverage": [ 43.57861328125, 27.04052734375, 15.97021484375 ] }, { "deviceUtilizationPercent": 54, "elapsedSeconds": 35.2240731716156, "loadAverage": [ 42.9716796875, 27.18896484375, 16.08740234375 ] }, { "deviceUtilizationPercent": 55, "elapsedSeconds": 37.419830083847046, "loadAverage": [ 42.9716796875, 27.18896484375, 16.08740234375 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 39.66512608528137, "loadAverage": [ 42.9716796875, 27.18896484375, 16.08740234375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 41.80920720100403, "loadAverage": [ 40.7314453125, 26.986328125, 16.08056640625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 43.933741092681885, "loadAverage": [ 40.7314453125, 26.986328125, 16.08056640625 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 46.0774941444397, "loadAverage": [ 38.0302734375, 26.654296875, 16.02734375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 48.19802713394165, "loadAverage": [ 38.0302734375, 26.654296875, 16.02734375 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 50.3784761428833, "loadAverage": [ 35.78564453125, 26.37744140625, 15.99169921875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 52.59400296211243, "loadAverage": [ 35.78564453125, 26.37744140625, 15.99169921875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 54.80043601989746, "loadAverage": [ 35.78564453125, 26.37744140625, 15.99169921875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 56.97805404663086, "loadAverage": [ 33.560546875, 26.072265625, 15.94482421875 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 59.15190625190735, "loadAverage": [ 33.560546875, 26.072265625, 15.94482421875 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9471770624, "processSeconds": 61.24811029434204, "profiled": false, "prompt": "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788625342.704321, "uncontended": false, "workload": "chat" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.1710532943249339 }, "endToEndSpeedups": { "adaptive": 1.139826179847793 }, "promptCharacters": 413, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 5.787040948867798, "decodeTokensPerSecond": 44.236770097520214, "elapsedSeconds": 5.953824996948242, "endToEndTokensPerSecond": 42.9975688118509, "expectedMTPActive": false, "firstTokenSeconds": 0.025865912437438965, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "baseline", "outputSHA256": "c8a6979d6467df2d9d8bf076069bd770aaf35e39ffbd0184230b4622e2be3ae3", "policy": "disabled", "prefillSeconds": 0.16346395015716553, "prefillTokensPerSecond": 593.4030096956393, "promptTokens": 97, "residentMemoryAfterBytes": 7706771456, "residentMemoryBeforeBytes": 7706591232, "ttftSeconds": 0.18934786319732666 }, { "acceleration": { "acceptanceRate": 0.2265625, "acceptedDraftTokens": 29, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 4.941740036010742, "decodeTokensPerSecond": 51.80361535299578, "elapsedSeconds": 5.223449945449829, "endToEndTokensPerSecond": 49.00975460155462, "expectedMTPActive": true, "firstTokenSeconds": 0.0011650323867797852, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "adaptive", "outputSHA256": "3de851f8ebd538e1ec22ad34d647ef21aefe55d11c4125329a8893293a21daab", "policy": "adaptive", "prefillSeconds": 0.2797049283981323, "prefillTokensPerSecond": 346.7940323952036, "promptTokens": 97, "residentMemoryAfterBytes": 9471770624, "residentMemoryBeforeBytes": 9471590400, "ttftSeconds": 0.280877947807312 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.6639627200302183 }, "endToEndSpeedups": { "adaptive": 1.6336748691410523 }, "promptCharacters": 413, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 9.581638097763062, "decodeTokensPerSecond": 26.717769695326524, "elapsedSeconds": 9.778503060340881, "endToEndTokensPerSecond": 26.179876246935056, "expectedMTPActive": false, "firstTokenSeconds": 0.033918023109436035, "generatedTokens": 256, "loadSeconds": 1.704692840576172e-05, "name": "baseline", "outputSHA256": "d192d769b30d7f0ba00d8325d1ea0d444cf8ce19004a8f593e5c0d3499e4bb2a", "policy": "disabled", "prefillSeconds": 0.19339895248413086, "prefillTokensPerSecond": 501.5539058204528, "promptTokens": 97, "residentMemoryAfterBytes": 7706853376, "residentMemoryBeforeBytes": 7706869760, "ttftSeconds": 0.22733402252197266 }, { "acceleration": { "acceptanceRate": 0.25, "acceptedDraftTokens": 32, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 5.758324980735779, "decodeTokensPerSecond": 44.45737273537646, "elapsedSeconds": 5.985587000846863, "endToEndTokensPerSecond": 42.76940590184057, "expectedMTPActive": true, "firstTokenSeconds": 0.0019429922103881836, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "adaptive", "outputSHA256": "c1a3962ad74497ad01f89c0245ea26e6c5c3bf1125010b761009d1489c206d4f", "policy": "adaptive", "prefillSeconds": 0.22514593601226807, "prefillTokensPerSecond": 430.83167175051534, "promptTokens": 97, "residentMemoryAfterBytes": 9471737856, "residentMemoryBeforeBytes": 9471770624, "ttftSeconds": 0.22709691524505615 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.9800947292274851 }, "endToEndSpeedups": { "adaptive": 0.9858870038453017 }, "promptCharacters": 413, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 4.8860780000686646, "decodeTokensPerSecond": 52.393760393592245, "elapsedSeconds": 5.071536898612976, "endToEndTokensPerSecond": 50.47779501910237, "expectedMTPActive": false, "firstTokenSeconds": 0.022485017776489258, "generatedTokens": 256, "loadSeconds": 1.4066696166992188e-05, "name": "baseline", "outputSHA256": "1436427f286030608a6040514f79a62a7857a007ddc7deeef87c3215821c232c", "policy": "disabled", "prefillSeconds": 0.18234801292419434, "prefillTokensPerSecond": 531.9498602944734, "promptTokens": 97, "residentMemoryAfterBytes": 7706886144, "residentMemoryBeforeBytes": 7706853376, "ttftSeconds": 0.20484709739685059 }, { "acceleration": { "acceptanceRate": 0.3671875, "acceptedDraftTokens": 47, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 4.985311985015869, "decodeTokensPerSecond": 51.35084840616752, "elapsedSeconds": 5.144136071205139, "endToEndTokensPerSecond": 49.76540209210013, "expectedMTPActive": true, "firstTokenSeconds": 0.0009720325469970703, "generatedTokens": 256, "loadSeconds": 1.1920928955078125e-05, "name": "adaptive", "outputSHA256": "deb0276d265c1c5d920740186a4a5eea31b8c8b240ebfe5900e99386506727f0", "policy": "adaptive", "prefillSeconds": 0.15602898597717285, "prefillTokensPerSecond": 621.6793590787751, "promptTokens": 97, "residentMemoryAfterBytes": 9471754240, "residentMemoryBeforeBytes": 9471737856, "ttftSeconds": 0.157012939453125 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-sampled/ornith-prose", "model": "text-agent-ornith-35b-mlx-4bit", "rawReceiptSHA256": "6848a36376a03d0aec32b0d5134443de42c0941e4461e6f3549c7525cd70a93d", "receipt": { "after": { "gpuDeviceUtilizationPercent": 64, "loadAverage": [ 19.76708984375, 23.4599609375, 15.451171875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7260.25M free = 931.75M (encrypted)", "swapUsedMiB": 7260.25 }, "before": { "gpuDeviceUtilizationPercent": 65, "loadAverage": [ 31.673828125, 25.80517578125, 15.90966796875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7268.25M free = 923.75M (encrypted)", "swapUsedMiB": 7268.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "text-agent-ornith-35b-mlx-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0.7", "--top-p", "0.9", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 66, "elapsedSeconds": 0.10001420974731445, "loadAverage": [ 31.673828125, 25.80517578125, 15.90966796875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 2.2100141048431396, "loadAverage": [ 31.673828125, 25.80517578125, 15.90966796875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 4.325017213821411, "loadAverage": [ 29.53759765625, 25.45947265625, 15.845703125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 6.465065002441406, "loadAverage": [ 29.53759765625, 25.45947265625, 15.845703125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 8.59277606010437, "loadAverage": [ 27.33203125, 25.06982421875, 15.76416015625 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 10.722113132476807, "loadAverage": [ 27.33203125, 25.06982421875, 15.76416015625 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 12.86846113204956, "loadAverage": [ 27.33203125, 25.06982421875, 15.76416015625 ] }, { "deviceUtilizationPercent": 67, "elapsedSeconds": 14.981714963912964, "loadAverage": [ 25.38330078125, 24.703125, 15.68896484375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 17.09501314163208, "loadAverage": [ 25.38330078125, 24.703125, 15.68896484375 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 19.271420001983643, "loadAverage": [ 23.5908203125, 24.3427734375, 15.6142578125 ] }, { "deviceUtilizationPercent": 65, "elapsedSeconds": 21.451090097427368, "loadAverage": [ 23.5908203125, 24.3427734375, 15.6142578125 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 23.571006059646606, "loadAverage": [ 22.10205078125, 24.021484375, 15.5517578125 ] }, { "deviceUtilizationPercent": 74, "elapsedSeconds": 25.690943002700806, "loadAverage": [ 22.10205078125, 24.021484375, 15.5517578125 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 27.8071551322937, "loadAverage": [ 22.10205078125, 24.021484375, 15.5517578125 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 29.9275860786438, "loadAverage": [ 21.052734375, 23.77197265625, 15.51318359375 ] }, { "deviceUtilizationPercent": 71, "elapsedSeconds": 32.04764008522034, "loadAverage": [ 21.052734375, 23.77197265625, 15.51318359375 ] } ], "knownRenderingWorkerCount": 0, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 9472868352, "processSeconds": 34.14914011955261, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788625404.169546, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.8168138827620122 }, "endToEndSpeedups": { "adaptive": 0.8172509842405772 }, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 3.3585410118103027, "decodeTokensPerSecond": 76.22357419479962, "elapsedSeconds": 3.4765350818634033, "endToEndTokensPerSecond": 73.63653579551553, "expectedMTPActive": false, "firstTokenSeconds": 0.00860297679901123, "generatedTokens": 256, "loadSeconds": 8.940696716308594e-06, "name": "baseline", "outputSHA256": "9c637eb44bc467e869cca2004c5264e75e5a57cc6bdbd7bdc650d1b02f3bef7c", "policy": "disabled", "prefillSeconds": 0.1161949634552002, "prefillTokensPerSecond": 516.3735003293274, "promptTokens": 60, "residentMemoryAfterBytes": 7686668288, "residentMemoryBeforeBytes": 7686307840, "ttftSeconds": 0.12480688095092773 }, { "acceleration": { "acceptanceRate": 0.1875, "acceptedDraftTokens": 24, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 4.11175799369812, "decodeTokensPerSecond": 62.2604735960526, "elapsedSeconds": 4.25393807888031, "endToEndTokensPerSecond": 60.17953135495156, "expectedMTPActive": true, "firstTokenSeconds": 0.001341104507446289, "generatedTokens": 256, "loadSeconds": 1.2993812561035156e-05, "name": "adaptive", "outputSHA256": "7278a5cd4efcbfd1d599abcc0a3150c07a1b07cef220e7c7e6f57d49dc33077c", "policy": "adaptive", "prefillSeconds": 0.13899600505828857, "prefillTokensPerSecond": 431.66708262470377, "promptTokens": 60, "residentMemoryAfterBytes": 9472720896, "residentMemoryBeforeBytes": 9472720896, "ttftSeconds": 0.1403501033782959 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.5973224910825723 }, "endToEndSpeedups": { "adaptive": 0.6158143223943471 }, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.415068030357361, "decodeTokensPerSecond": 106.00115474267585, "elapsedSeconds": 2.5554779767990112, "endToEndTokensPerSecond": 100.17695410573066, "expectedMTPActive": false, "firstTokenSeconds": 0.008687973022460938, "generatedTokens": 256, "loadSeconds": 8.940696716308594e-06, "name": "baseline", "outputSHA256": "62edf067f1420f338e07e2eabef1aff78e970ee722971f6b56028249210c6a5c", "policy": "disabled", "prefillSeconds": 0.13842999935150146, "prefillTokensPerSecond": 433.43206155515463, "promptTokens": 60, "residentMemoryAfterBytes": 7686864896, "residentMemoryBeforeBytes": 7686766592, "ttftSeconds": 0.1471269130706787 }, { "acceleration": { "acceptanceRate": 0.1640625, "acceptedDraftTokens": 21, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 4.043156027793884, "decodeTokensPerSecond": 63.31687380852436, "elapsedSeconds": 4.149754047393799, "endToEndTokensPerSecond": 61.69040311215013, "expectedMTPActive": true, "firstTokenSeconds": 0.0007259845733642578, "generatedTokens": 256, "loadSeconds": 7.033348083496094e-06, "name": "adaptive", "outputSHA256": "5557f1ef4138505a8fd9854828fa96d3a2e55f8de14d9165aecfea20d3c4582d", "policy": "adaptive", "prefillSeconds": 0.10468709468841553, "prefillTokensPerSecond": 573.1365473326053, "promptTokens": 60, "residentMemoryAfterBytes": 9472835584, "residentMemoryBeforeBytes": 9472737280, "ttftSeconds": 0.10542011260986328 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 0.7585506573085363 }, "endToEndSpeedups": { "adaptive": 0.7730849449724378 }, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 2.8538800477981567, "decodeTokensPerSecond": 89.70243868431356, "elapsedSeconds": 2.995473027229309, "endToEndTokensPerSecond": 85.46229516103826, "expectedMTPActive": false, "firstTokenSeconds": 0.01191103458404541, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "baseline", "outputSHA256": "aa8ca721886760d8e4bc11751ff818db7077d19cf8df9d53e1e5de543324f16d", "policy": "disabled", "prefillSeconds": 0.1395360231399536, "prefillTokensPerSecond": 429.99648871904884, "promptTokens": 60, "residentMemoryAfterBytes": 7686897664, "residentMemoryBeforeBytes": 7686864896, "ttftSeconds": 0.15145504474639893 }, { "acceleration": { "acceptanceRate": 0.1953125, "acceptedDraftTokens": 25, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 3.7622801065444946, "decodeTokensPerSecond": 68.04384382616473, "elapsedSeconds": 3.8747010231018066, "endToEndTokensPerSecond": 66.0696137517895, "expectedMTPActive": true, "firstTokenSeconds": 0.0007650852203369141, "generatedTokens": 256, "loadSeconds": 7.987022399902344e-06, "name": "adaptive", "outputSHA256": "8e412cc844555ac899bd0c6a6476cf23f22c68fe878f1334c11530b114af47b4", "policy": "adaptive", "prefillSeconds": 0.11035001277923584, "prefillTokensPerSecond": 543.724449946688, "promptTokens": 60, "residentMemoryAfterBytes": 9472868352, "residentMemoryBeforeBytes": 9472835584, "ttftSeconds": 0.11112308502197266 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-sampled/qwen-chat", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "0a24489464d2fea6ef659d224bea6c6af7bca44c827a31bcb8c41069cc0b35ed", "receipt": { "after": { "gpuDeviceUtilizationPercent": 74, "loadAverage": [ 30.96826171875, 31.13037109375, 20.046875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7260.25M free = 931.75M (encrypted)", "swapUsedMiB": 7260.25 }, "before": { "gpuDeviceUtilizationPercent": 35, "loadAverage": [ 19.76708984375, 23.4599609375, 15.451171875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7260.25M free = 931.75M (encrypted)", "swapUsedMiB": 7260.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0.7", "--top-p", "0.9", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 34, "elapsedSeconds": 0.09610867500305176, "loadAverage": [ 19.76708984375, 23.4599609375, 15.451171875 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 2.2024757862091064, "loadAverage": [ 19.76708984375, 23.4599609375, 15.451171875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 4.309813022613525, "loadAverage": [ 18.74462890625, 23.1865234375, 15.4013671875 ] }, { "deviceUtilizationPercent": 49, "elapsedSeconds": 6.426228046417236, "loadAverage": [ 18.74462890625, 23.1865234375, 15.4013671875 ] }, { "deviceUtilizationPercent": 40, "elapsedSeconds": 8.543068885803223, "loadAverage": [ 18.74462890625, 23.1865234375, 15.4013671875 ] }, { "deviceUtilizationPercent": 42, "elapsedSeconds": 10.650874853134155, "loadAverage": [ 17.2431640625, 22.80126953125, 15.31103515625 ] }, { "deviceUtilizationPercent": 46, "elapsedSeconds": 12.766998052597046, "loadAverage": [ 17.2431640625, 22.80126953125, 15.31103515625 ] }, { "deviceUtilizationPercent": 37, "elapsedSeconds": 14.880131721496582, "loadAverage": [ 16.1025390625, 22.47216796875, 15.23876953125 ] }, { "deviceUtilizationPercent": 43, "elapsedSeconds": 17.00480079650879, "loadAverage": [ 16.1025390625, 22.47216796875, 15.23876953125 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 19.116302728652954, "loadAverage": [ 15.13330078125, 22.1650390625, 15.1728515625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 22.17778205871582, "loadAverage": [ 15.13330078125, 22.1650390625, 15.1728515625 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 25.07042098045349, "loadAverage": [ 20.40771484375, 23.1416015625, 15.55810546875 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 27.768784761428833, "loadAverage": [ 20.40771484375, 23.1416015625, 15.55810546875 ] }, { "deviceUtilizationPercent": 63, "elapsedSeconds": 30.382980823516846, "loadAverage": [ 21.255859375, 23.27197265625, 15.6484375 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 32.970921993255615, "loadAverage": [ 21.255859375, 23.27197265625, 15.6484375 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 35.55562686920166, "loadAverage": [ 27.8017578125, 24.59521484375, 16.16015625 ] }, { "deviceUtilizationPercent": 62, "elapsedSeconds": 38.126168727874756, "loadAverage": [ 27.8017578125, 24.59521484375, 16.16015625 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 40.74195885658264, "loadAverage": [ 35.1845703125, 26.1787109375, 16.7685546875 ] }, { "deviceUtilizationPercent": 59, "elapsedSeconds": 43.40412974357605, "loadAverage": [ 35.1845703125, 26.1787109375, 16.7685546875 ] }, { "deviceUtilizationPercent": 57, "elapsedSeconds": 46.04906868934631, "loadAverage": [ 43.17724609375, 27.98486328125, 17.4609375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 48.768786907196045, "loadAverage": [ 43.17724609375, 27.98486328125, 17.4609375 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 50.89795994758606, "loadAverage": [ 40.11962890625, 27.60302734375, 17.3876953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 53.03210687637329, "loadAverage": [ 40.11962890625, 27.60302734375, 17.3876953125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 55.26501679420471, "loadAverage": [ 37.06689453125, 27.177734375, 17.29736328125 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 57.66056776046753, "loadAverage": [ 37.06689453125, 27.177734375, 17.29736328125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 60.14572596549988, "loadAverage": [ 40.0244140625, 27.95458984375, 17.62939453125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 62.64059805870056, "loadAverage": [ 40.0244140625, 27.95458984375, 17.62939453125 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 65.1672739982605, "loadAverage": [ 44.02587890625, 28.984375, 18.05322265625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 67.45871996879578, "loadAverage": [ 44.02587890625, 28.984375, 18.05322265625 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 70.02162098884583, "loadAverage": [ 47.546875, 29.9638671875, 18.462890625 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 72.77603197097778, "loadAverage": [ 47.546875, 29.9638671875, 18.462890625 ] }, { "deviceUtilizationPercent": 73, "elapsedSeconds": 75.50239491462708, "loadAverage": [ 56.8720703125, 32.18896484375, 19.3154296875 ] }, { "deviceUtilizationPercent": 79, "elapsedSeconds": 78.14628887176514, "loadAverage": [ 56.8720703125, 32.18896484375, 19.3154296875 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 80.80446791648865, "loadAverage": [ 64.1689453125, 34.111328125, 20.0693359375 ] }, { "deviceUtilizationPercent": 80, "elapsedSeconds": 83.42289471626282, "loadAverage": [ 64.1689453125, 34.111328125, 20.0693359375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 85.86873388290405, "loadAverage": [ 69.7607421875, 35.76953125, 20.73681640625 ] }, { "deviceUtilizationPercent": 61, "elapsedSeconds": 88.15562295913696, "loadAverage": [ 69.7607421875, 35.76953125, 20.73681640625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 90.44096183776855, "loadAverage": [ 69.53955078125, 36.28759765625, 21.0078125 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 92.71558570861816, "loadAverage": [ 69.53955078125, 36.28759765625, 21.0078125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 94.8958158493042, "loadAverage": [ 65.572265625, 36.01708984375, 21.00146484375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 97.03430390357971, "loadAverage": [ 65.572265625, 36.01708984375, 21.00146484375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 99.14502882957458, "loadAverage": [ 60.7216796875, 35.501953125, 20.9072265625 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 101.24752998352051, "loadAverage": [ 60.7216796875, 35.501953125, 20.9072265625 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 103.35191774368286, "loadAverage": [ 60.7216796875, 35.501953125, 20.9072265625 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 105.46239686012268, "loadAverage": [ 56.09912109375, 34.9619140625, 20.80224609375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 107.57416296005249, "loadAverage": [ 56.09912109375, 34.9619140625, 20.80224609375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 109.6816828250885, "loadAverage": [ 51.7666015625, 34.41455078125, 20.69189453125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 111.78833270072937, "loadAverage": [ 51.7666015625, 34.41455078125, 20.69189453125 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 113.90815377235413, "loadAverage": [ 51.7666015625, 34.41455078125, 20.69189453125 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 116.01881980895996, "loadAverage": [ 47.861328125, 33.892578125, 20.587890625 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 118.1219847202301, "loadAverage": [ 47.861328125, 33.892578125, 20.587890625 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 120.23059892654419, "loadAverage": [ 44.3486328125, 33.39599609375, 20.490234375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 122.33595395088196, "loadAverage": [ 44.3486328125, 33.39599609375, 20.490234375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 124.4567129611969, "loadAverage": [ 41.27734375, 32.94091796875, 20.4052734375 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 126.5682020187378, "loadAverage": [ 41.27734375, 32.94091796875, 20.4052734375 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 128.67641186714172, "loadAverage": [ 41.27734375, 32.94091796875, 20.4052734375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 130.79330897331238, "loadAverage": [ 38.4521484375, 32.49365234375, 20.32080078125 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 132.90493178367615, "loadAverage": [ 38.4521484375, 32.49365234375, 20.32080078125 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 135.03748202323914, "loadAverage": [ 35.85302734375, 32.0537109375, 20.23681640625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 137.23707795143127, "loadAverage": [ 35.85302734375, 32.0537109375, 20.23681640625 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 139.48133993148804, "loadAverage": [ 33.14208984375, 31.5546875, 20.1298828125 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 141.90272688865662, "loadAverage": [ 33.14208984375, 31.5546875, 20.1298828125 ] }, { "deviceUtilizationPercent": 70, "elapsedSeconds": 144.35601162910461, "loadAverage": [ 30.96826171875, 31.13037109375, 20.046875 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15822356480, "processSeconds": 146.51112294197083, "profiled": false, "prompt": "I'm trying to get back into reading after a busy year. I have about twenty minutes most evenings, enjoy mysteries and science fiction, and keep abandoning books when I miss a few days. Help me make a gentle, realistic four-week plan. Talk to me conversationally, suggest ways to choose books without buying a stack, and explain what to do when I fall behind. Give enough practical detail that I can start tonight.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788625438.5240471, "uncontended": false, "workload": "chat" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 2.3077025055929505 }, "endToEndSpeedups": { "adaptive": 2.1891414517251944 }, "promptCharacters": 413, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 28.18172299861908, "decodeTokensPerSecond": 9.083901648332295, "elapsedSeconds": 29.62769591808319, "endToEndTokensPerSecond": 8.640563907089078, "expectedMTPActive": false, "firstTokenSeconds": 0.06806004047393799, "generatedTokens": 256, "loadSeconds": 3.0994415283203125e-05, "name": "baseline", "outputSHA256": "030943934673e47f89f279eb31e426f1569c10ae0f088ba8f24977c6f4f96a36", "policy": "disabled", "prefillSeconds": 1.4430179595947266, "prefillTokensPerSecond": 67.22023059729803, "promptTokens": 97, "residentMemoryAfterBytes": 15425191936, "residentMemoryBeforeBytes": 15425060864, "ttftSeconds": 1.5111089944839478 }, { "acceleration": { "acceptanceRate": 0.734375, "acceptedDraftTokens": 94, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 12.212026000022888, "decodeTokensPerSecond": 20.962942594416372, "elapsedSeconds": 13.533933997154236, "endToEndTokensPerSecond": 18.915416615289303, "expectedMTPActive": true, "firstTokenSeconds": 0.0012871026992797852, "generatedTokens": 256, "loadSeconds": 2.110004425048828e-05, "name": "adaptive", "outputSHA256": "2960122f27932e23a40c1245074765bc2c1f253a7cebce8a0a4824516c0f0206", "policy": "adaptive", "prefillSeconds": 1.3202329874038696, "prefillTokensPerSecond": 73.47188028587483, "promptTokens": 97, "residentMemoryAfterBytes": 15822159872, "residentMemoryBeforeBytes": 15822143488, "ttftSeconds": 1.3215411901474 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.295944099143189 }, "endToEndSpeedups": { "adaptive": 1.2866043340925313 }, "promptCharacters": 413, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.61055600643158, "decodeTokensPerSecond": 15.411886266834008, "elapsedSeconds": 18.426818013191223, "endToEndTokensPerSecond": 13.892794719996532, "expectedMTPActive": false, "firstTokenSeconds": 0.08396899700164795, "generatedTokens": 256, "loadSeconds": 4.601478576660156e-05, "name": "baseline", "outputSHA256": "2f41865da94d1bf783b8a2e0c3c0c5a2d1ff6beabb30c0dca76084b28c587536", "policy": "disabled", "prefillSeconds": 1.8113800287246704, "prefillTokensPerSecond": 53.55033094203557, "promptTokens": 97, "residentMemoryAfterBytes": 15425404928, "residentMemoryBeforeBytes": 15425306624, "ttftSeconds": 1.895395040512085 }, { "acceleration": { "acceptanceRate": 0.703125, "acceptedDraftTokens": 90, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 12.817339897155762, "decodeTokensPerSecond": 19.972943064169485, "elapsedSeconds": 14.322054982185364, "endToEndTokensPerSecond": 17.874529899405374, "expectedMTPActive": true, "firstTokenSeconds": 0.0024329423904418945, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "adaptive", "outputSHA256": "f7918e68f9608c6e8ff1085a552b9ae62ad0da48ee4a39d2523842e5ba7d5fb4", "policy": "adaptive", "prefillSeconds": 1.5028769969940186, "prefillTokensPerSecond": 64.54287356451304, "promptTokens": 97, "residentMemoryAfterBytes": 15822290944, "residentMemoryBeforeBytes": 15822176256, "ttftSeconds": 1.5053359270095825 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.4218051498383406 }, "endToEndSpeedups": { "adaptive": 1.4056751270498349 }, "promptCharacters": 413, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 17.938609957695007, "decodeTokensPerSecond": 14.270893932346489, "elapsedSeconds": 19.537933945655823, "endToEndTokensPerSecond": 13.10271601450063, "expectedMTPActive": false, "firstTokenSeconds": 0.14965295791625977, "generatedTokens": 256, "loadSeconds": 5.4001808166503906e-05, "name": "baseline", "outputSHA256": "af2b3923f060aa46e13e5c2909129e3ad1a31d4dfc53919688950f1439773e82", "policy": "disabled", "prefillSeconds": 1.59026300907135, "prefillTokensPerSecond": 60.996199651681586, "promptTokens": 97, "residentMemoryAfterBytes": 15425421312, "residentMemoryBeforeBytes": 15425404928, "ttftSeconds": 1.7399699687957764 }, { "acceleration": { "acceptanceRate": 0.7421875, "acceptedDraftTokens": 95, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 12.616785049438477, "decodeTokensPerSecond": 20.290430485806965, "elapsedSeconds": 13.8993239402771, "endToEndTokensPerSecond": 18.418161998381077, "expectedMTPActive": true, "firstTokenSeconds": 0.0010840892791748047, "generatedTokens": 256, "loadSeconds": 1.800060272216797e-05, "name": "adaptive", "outputSHA256": "d20fd0828ec9afeed0c296cd394be86561240544d9230c57a04297df0fea7233", "policy": "adaptive", "prefillSeconds": 1.2787469625473022, "prefillTokensPerSecond": 75.85550765006167, "promptTokens": 97, "residentMemoryAfterBytes": 15822356480, "residentMemoryBeforeBytes": 15822290944, "ttftSeconds": 1.2798490524291992 } ], "warmupTokens": 256, "warmups": 1 } ] }, { "id": "final-sampled/qwen-prose", "model": "vision-chat-q38-27b-4bit", "rawReceiptSHA256": "1770b1bd8bf7ee1271fe9ebfb41ad61bb2822573412631c611bc3cfc1d913188", "receipt": { "after": { "gpuDeviceUtilizationPercent": 48, "loadAverage": [ 11.69970703125, 23.03759765625, 18.41015625 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7260.25M free = 931.75M (encrypted)", "swapUsedMiB": 7260.25 }, "before": { "gpuDeviceUtilizationPercent": 34, "loadAverage": [ 30.96826171875, 31.13037109375, 20.046875 ], "pressureLevel": 1, "swap": "total = 8192.00M used = 7260.25M free = 931.75M (encrypted)", "swapUsedMiB": 7260.25 }, "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "buildSourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "collectorSHA256": "dec1ae60840c738d2c8ef678adf8c1fc39b7eecaab018032ff2c596b51599f2c", "command": [ "mere.run", "model", "benchmark", "q36-mtp", "--model", "vision-chat-q38-27b-4bit", "--prompt", "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "--decode-tokens", "256", "--context-size", "8192", "--temperature", "0.7", "--top-p", "0.9", "--json", "--variants", "baseline,adaptive", "--warmups", "1", "--warmup-tokens", "256", "--repetitions", "3" ], "competingInferenceProcessCount": 0, "gpuSamples": [ { "deviceUtilizationPercent": 34, "elapsedSeconds": 0.12845993041992188, "loadAverage": [ 30.96826171875, 31.13037109375, 20.046875 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 2.267130136489868, "loadAverage": [ 30.96826171875, 31.13037109375, 20.046875 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 4.552784204483032, "loadAverage": [ 30.169921875, 30.9619140625, 20.05224609375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 7.190841197967529, "loadAverage": [ 30.169921875, 30.9619140625, 20.05224609375 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 10.172383069992065, "loadAverage": [ 28.154296875, 30.53076171875, 19.9638671875 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 12.505019903182983, "loadAverage": [ 26.5400390625, 30.15625, 19.8935546875 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 14.7086820602417, "loadAverage": [ 26.5400390625, 30.15625, 19.8935546875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 16.84237003326416, "loadAverage": [ 26.5400390625, 30.15625, 19.8935546875 ] }, { "deviceUtilizationPercent": 98, "elapsedSeconds": 19.27529287338257, "loadAverage": [ 24.89501953125, 29.7548828125, 19.81201171875 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 22.054347038269043, "loadAverage": [ 24.89501953125, 29.7548828125, 19.81201171875 ] }, { "deviceUtilizationPercent": 75, "elapsedSeconds": 24.89551615715027, "loadAverage": [ 25.3837890625, 29.775390625, 19.87744140625 ] }, { "deviceUtilizationPercent": 76, "elapsedSeconds": 27.4933021068573, "loadAverage": [ 24.47216796875, 29.51318359375, 19.8427734375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 30.2139630317688, "loadAverage": [ 24.47216796875, 29.51318359375, 19.8427734375 ] }, { "deviceUtilizationPercent": 78, "elapsedSeconds": 33.09423303604126, "loadAverage": [ 24.91455078125, 29.52099609375, 19.90185546875 ] }, { "deviceUtilizationPercent": 94, "elapsedSeconds": 35.37997484207153, "loadAverage": [ 24.91455078125, 29.52099609375, 19.90185546875 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 37.51497292518616, "loadAverage": [ 23.39990234375, 29.13037109375, 19.8203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 39.63887000083923, "loadAverage": [ 23.39990234375, 29.13037109375, 19.8203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 41.767460107803345, "loadAverage": [ 23.39990234375, 29.13037109375, 19.8203125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 43.88966608047485, "loadAverage": [ 21.76611328125, 28.6962890625, 19.7216796875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 46.00091600418091, "loadAverage": [ 21.76611328125, 28.6962890625, 19.7216796875 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 48.119051933288574, "loadAverage": [ 20.34326171875, 28.2861328125, 19.62939453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 50.23794412612915, "loadAverage": [ 20.34326171875, 28.2861328125, 19.62939453125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 52.3533570766449, "loadAverage": [ 18.9541015625, 27.8662109375, 19.53173828125 ] }, { "deviceUtilizationPercent": 99, "elapsedSeconds": 54.47069692611694, "loadAverage": [ 18.9541015625, 27.8662109375, 19.53173828125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 56.58545994758606, "loadAverage": [ 18.9541015625, 27.8662109375, 19.53173828125 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 58.710379123687744, "loadAverage": [ 17.91650390625, 27.5029296875, 19.4521484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 60.82476806640625, "loadAverage": [ 17.91650390625, 27.5029296875, 19.4521484375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 62.93484115600586, "loadAverage": [ 16.5615234375, 27.0625, 19.34375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 65.05341601371765, "loadAverage": [ 16.5615234375, 27.0625, 19.34375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 67.16626000404358, "loadAverage": [ 16.5615234375, 27.0625, 19.34375 ] }, { "deviceUtilizationPercent": 100, "elapsedSeconds": 69.28089809417725, "loadAverage": [ 15.39501953125, 26.64599609375, 19.24169921875 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 71.38575315475464, "loadAverage": [ 15.39501953125, 26.64599609375, 19.24169921875 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 73.5024061203003, "loadAverage": [ 14.322265625, 26.23681640625, 19.140625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 75.62739109992981, "loadAverage": [ 14.322265625, 26.23681640625, 19.140625 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 77.73383092880249, "loadAverage": [ 13.73583984375, 25.9169921875, 19.0693359375 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 79.83405590057373, "loadAverage": [ 13.73583984375, 25.9169921875, 19.0693359375 ] }, { "deviceUtilizationPercent": 83, "elapsedSeconds": 81.94128012657166, "loadAverage": [ 13.73583984375, 25.9169921875, 19.0693359375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 84.04478788375854, "loadAverage": [ 12.7158203125, 25.5029296875, 18.96337890625 ] }, { "deviceUtilizationPercent": 97, "elapsedSeconds": 86.16982102394104, "loadAverage": [ 12.7158203125, 25.5029296875, 18.96337890625 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 88.27676010131836, "loadAverage": [ 11.9375, 25.12890625, 18.86962890625 ] }, { "deviceUtilizationPercent": 92, "elapsedSeconds": 90.38184404373169, "loadAverage": [ 11.9375, 25.12890625, 18.86962890625 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 92.48637890815735, "loadAverage": [ 11.1416015625, 24.74462890625, 18.7705078125 ] }, { "deviceUtilizationPercent": 88, "elapsedSeconds": 94.59166216850281, "loadAverage": [ 11.1416015625, 24.74462890625, 18.7705078125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 96.69545125961304, "loadAverage": [ 11.1416015625, 24.74462890625, 18.7705078125 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 98.80967092514038, "loadAverage": [ 10.7294921875, 24.43310546875, 18.6953125 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 100.94465708732605, "loadAverage": [ 10.7294921875, 24.43310546875, 18.6953125 ] }, { "deviceUtilizationPercent": 96, "elapsedSeconds": 103.07289814949036, "loadAverage": [ 10.0302734375, 24.060546875, 18.59716796875 ] }, { "deviceUtilizationPercent": 90, "elapsedSeconds": 105.25833797454834, "loadAverage": [ 10.0302734375, 24.060546875, 18.59716796875 ] }, { "deviceUtilizationPercent": 93, "elapsedSeconds": 107.3992350101471, "loadAverage": [ 10.42822265625, 23.90966796875, 18.57568359375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 109.55516314506531, "loadAverage": [ 10.42822265625, 23.90966796875, 18.57568359375 ] }, { "deviceUtilizationPercent": 85, "elapsedSeconds": 111.68390083312988, "loadAverage": [ 10.42822265625, 23.90966796875, 18.57568359375 ] }, { "deviceUtilizationPercent": 84, "elapsedSeconds": 113.81305694580078, "loadAverage": [ 9.8330078125, 23.5625, 18.484375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 115.9674301147461, "loadAverage": [ 9.8330078125, 23.5625, 18.484375 ] }, { "deviceUtilizationPercent": 95, "elapsedSeconds": 118.08792495727539, "loadAverage": [ 9.44580078125, 23.25390625, 18.4052734375 ] }, { "deviceUtilizationPercent": 89, "elapsedSeconds": 120.32083821296692, "loadAverage": [ 9.44580078125, 23.25390625, 18.4052734375 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 122.51242113113403, "loadAverage": [ 9.32958984375, 23.00048828125, 18.34423828125 ] }, { "deviceUtilizationPercent": 86, "elapsedSeconds": 124.64841604232788, "loadAverage": [ 9.32958984375, 23.00048828125, 18.34423828125 ] }, { "deviceUtilizationPercent": 87, "elapsedSeconds": 126.78138709068298, "loadAverage": [ 9.32958984375, 23.00048828125, 18.34423828125 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 128.93759989738464, "loadAverage": [ 9.0625, 22.7177734375, 18.271484375 ] }, { "deviceUtilizationPercent": 82, "elapsedSeconds": 131.08228087425232, "loadAverage": [ 9.0625, 22.7177734375, 18.271484375 ] }, { "deviceUtilizationPercent": 91, "elapsedSeconds": 133.4050612449646, "loadAverage": [ 11.69970703125, 23.03759765625, 18.41015625 ] } ], "knownRenderingWorkerCount": 1, "maxPressureLevel": 1, "measurementMode": "desktop-load", "peakResidentBytes": 15813574656, "processSeconds": 135.5230951309204, "profiled": false, "prompt": "Explain how a small community could turn an abandoned railway station into a public library. Discuss the building, accessibility, staffing, the collection, and a realistic opening-day scene. Write approximately 400 words of clear, connected prose.", "runtimeOverrides": {}, "sourceHead": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "startedUnixSeconds": 1788625585.329428, "uncontended": false, "workload": "prose" }, "scenarios": [ { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.194476340097812 }, "endToEndSpeedups": { "adaptive": 1.1644696775327341 }, "promptCharacters": 247, "repetition": 1, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 17.37777090072632, "decodeTokensPerSecond": 14.731463630315224, "elapsedSeconds": 18.300414085388184, "endToEndTokensPerSecond": 13.988754506074326, "expectedMTPActive": false, "firstTokenSeconds": 0.05843091011047363, "generatedTokens": 256, "loadSeconds": 3.3020973205566406e-05, "name": "baseline", "outputSHA256": "fd840c9ce0dded20126c282d297184ec253d65dcc0220bf2d7f0b33f431fb214", "policy": "disabled", "prefillSeconds": 0.9197919368743896, "prefillTokensPerSecond": 65.23214391712355, "promptTokens": 60, "residentMemoryAfterBytes": 15416639488, "residentMemoryBeforeBytes": 15416492032, "ttftSeconds": 0.9782558679580688 }, { "acceleration": { "acceptanceRate": 0.59375, "acceptedDraftTokens": 76, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 14.548442959785461, "decodeTokensPerSecond": 17.596384761422957, "elapsedSeconds": 15.71566390991211, "endToEndTokensPerSecond": 16.289480448772952, "expectedMTPActive": true, "firstTokenSeconds": 0.0017069578170776367, "generatedTokens": 256, "loadSeconds": 2.09808349609375e-05, "name": "adaptive", "outputSHA256": "f895946b0ceb6caedd3cf2a123ad3b030d5d939fc21d9174ecc5a6f1c063b6d1", "policy": "adaptive", "prefillSeconds": 1.1648629903793335, "prefillTokensPerSecond": 51.50820353598942, "promptTokens": 60, "residentMemoryAfterBytes": 15813525504, "residentMemoryBeforeBytes": 15813443584, "ttftSeconds": 1.166590929031372 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.0662801923110918 }, "endToEndSpeedups": { "adaptive": 1.0587489328114812 }, "promptCharacters": 247, "repetition": 2, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.027796030044556, "decodeTokensPerSecond": 15.972252174916674, "elapsedSeconds": 16.82800006866455, "endToEndTokensPerSecond": 15.212740608237699, "expectedMTPActive": false, "firstTokenSeconds": 0.04798603057861328, "generatedTokens": 256, "loadSeconds": 4.303455352783203e-05, "name": "baseline", "outputSHA256": "6cfce9b7b8d08c57b68c1156be005d86931fb1108c883cdf561f59429e0bf306", "policy": "disabled", "prefillSeconds": 0.7972429990768433, "prefillTokensPerSecond": 75.25936266543097, "promptTokens": 60, "residentMemoryAfterBytes": 15416901632, "residentMemoryBeforeBytes": 15416770560, "ttftSeconds": 0.8452720642089844 }, { "acceleration": { "acceptanceRate": 0.609375, "acceptedDraftTokens": 78, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 15.03150498867035, "decodeTokensPerSecond": 17.030896120711407, "elapsedSeconds": 15.894230961799622, "endToEndTokensPerSecond": 16.106472884109547, "expectedMTPActive": true, "firstTokenSeconds": 0.0013859272003173828, "generatedTokens": 256, "loadSeconds": 3.3974647521972656e-05, "name": "adaptive", "outputSHA256": "293ac99a5c8a78ab366c88082071b654a9954eb9a39004aa4b10b68a8d19306b", "policy": "adaptive", "prefillSeconds": 0.8606880903244019, "prefillTokensPerSecond": 69.71166520659698, "promptTokens": 60, "residentMemoryAfterBytes": 15813509120, "residentMemoryBeforeBytes": 15813525504, "ttftSeconds": 0.8621079921722412 } ], "warmupTokens": 256, "warmups": 1 }, { "contextSize": 8192, "decodeSpeedups": { "adaptive": 1.061922776422489 }, "endToEndSpeedups": { "adaptive": 1.0694411856037307 }, "promptCharacters": 247, "repetition": 3, "requestedDecodeTokens": 256, "temperature": 0.7, "variants": [ { "acceleration": { "route": "final-target-pipelined" }, "decodeSeconds": 16.32183802127838, "decodeTokensPerSecond": 15.684508060076265, "elapsedSeconds": 17.259183049201965, "endToEndTokensPerSecond": 14.832683521010399, "expectedMTPActive": false, "firstTokenSeconds": 0.056159019470214844, "generatedTokens": 256, "loadSeconds": 2.300739288330078e-05, "name": "baseline", "outputSHA256": "0e7ce9825ea4f93667d9eca5869f55c46370e1d65371a1c2bbdbaa979ca6a738", "policy": "disabled", "prefillSeconds": 0.9345680475234985, "prefillTokensPerSecond": 64.20078255295945, "promptTokens": 60, "residentMemoryAfterBytes": 15417032704, "residentMemoryBeforeBytes": 15416901632, "ttftSeconds": 0.9907500743865967 }, { "acceleration": { "acceptanceRate": 0.625, "acceptedDraftTokens": 80, "draftHistoryTokens": 0, "draftModel": "qwen-mtp", "draftedTokens": 128, "rounds": 128, "route": "mtp-speculative" }, "decodeSeconds": 15.37007999420166, "decodeTokensPerSecond": 16.655736345977093, "elapsedSeconds": 16.138506054878235, "endToEndTokensPerSecond": 15.862682650394278, "expectedMTPActive": true, "firstTokenSeconds": 0.0013599395751953125, "generatedTokens": 256, "loadSeconds": 2.5987625122070312e-05, "name": "adaptive", "outputSHA256": "8d7c098af3a84cf1f2f0ee117ac987aab2d2e7b7a43867e85f662d1601da970b", "policy": "adaptive", "prefillSeconds": 0.7659449577331543, "prefillTokensPerSecond": 78.33461059339365, "promptTokens": 60, "residentMemoryAfterBytes": 15813574656, "residentMemoryBeforeBytes": 15813525504, "ttftSeconds": 0.7673308849334717 } ], "warmupTokens": 256, "warmups": 1 } ] } ], "releaseBuild": { "binarySHA256": "37ae108285410acec9ee3863de69475fb74ce9b472bf23c05e0e0df8ce26414c", "command": "swift build -c release --product mere.run --disable-index-store", "completedUnixSeconds": 1788623568.49564, "sourceRevision": "c3ef55db5f09b2bcc6526bec4dd2a78eaa27aee6", "swiftVersion": "Apple Swift version 6.3 (swiftlang-6.3.0.123.5 clang-2100.0.123.102)\nTarget: arm64-apple-macosx26.0" }, "schemaVersion": 1, "validation": { "localGate": { "logSHA256": "bf60ac01795cda368cb89b4291f76c674738891648e9146738a4299c419cf625", "passed": true, "skipped": 309, "swiftTestingChecks": 51, "xctestCases": 3909 }, "ornithCheckpoint": { "logSHA256": "fa9c5faf6540a49c4a1d772fdfdc7c387008c00edf34fac15e50ab63bf7e37fc", "passed": true }, "qwenCheckpoint": { "logSHA256": "4eac1ffe41ed31323ddd21e95c034981c723aa0435205a30cbd7c12292041f1e", "passed": true }, "targetedMetal": { "logSHA256": "9bd330a41ffcb9d9cf93d63dbe789ada7bd44d8d5f8990e2d0014a6e3fafcfc4", "passed": true, "tests": 8 } } }