{
  "title": "On-device dictation cleanup models, compared on one sealed exam",
  "run_date": "2026-08-26",
  "publisher": {
    "name": "Envious Labs LLC",
    "product": "EnviousWispr",
    "url": "https://enviouswispr.com"
  },
  "conflict_of_interest": "The publisher builds EnviousWispr and its EG-1 model, which is one of the systems under test and the one that scored highest. The exam, the answer keys and the grading configuration are also the publisher's. Weigh the results accordingly. The mitigations that exist are listed under method.answer_keys and method.grading; the one that does not exist is independent selection of the test cases.",
  "task_under_test": "Turning a raw speech-to-text transcript into clean written text: removing filler, resolving self-corrections to what the speaker settled on, applying punctuation and capitalisation, formatting numbers, dates and addresses, and laying out spoken lists. This is post-processing of text. It is NOT speech recognition, and none of these models transcribe audio.",
  "method": {
    "corpus": {
      "id": "sealed_v1",
      "cases": 1462,
      "language": "English",
      "sealed_on": "2026-08-15",
      "case_shapes": {
        "clean": 309,
        "self_correction": 219,
        "fillers_only": 200,
        "voice_at_risk": 134,
        "unfinished": 120,
        "spoken_list": 114,
        "topic_shift": 102,
        "inline_enumeration": 87,
        "connected_prose": 74,
        "numbers_dates": 73,
        "quoted_instruction": 30
      },
      "provenance": "Scenarios generated to a production-weighted mix, spoken through text-to-speech, then transcribed by a real recogniser so the inputs carry real recognition errors."
    },
    "answer_keys": "Two models from different vendors independently authored answer keys from a policy written before the benchmark. Initial agreement was 77.94%; disagreements were adjudicated against the policy and 11 undecidable cases were excluded.",
    "grading": "The same GPT-5.6-luna judge model and rubric were used across these arms, with a second adjudication pass. Stored run summaries report judge_blind=false; masking of model identity is not independently verified here.",
    "prompting": "Each model was run with ITS OWN instructions, never a shared prompt: these are instruction-tuned for one specific wrapper, so a shared prompt measures the prompt. Sources are given per model under models[].prompt_source.",
    "decoding": "Greedy (temperature 0) for every model.",
    "hardware": "Apple M5 Max, 64 GB, macOS 26.",
    "latency_note": "Latency is single-stream: one dictation at a time, which is how a person uses these. Each model runs in the runtime its own vendor ships it in, so the figure includes the runtime. Earlier concurrent measurements were throughput, not latency, and are not reported here.",
    "metrics": {
      "pass_pct": "Share of cases judged clean or with a trivial blemish (pass + minor).",
      "serious_errors": "Count of cases where meaning was changed or content was lost (severity S4). Lower is better. Read alongside pass_pct: a model can pass less often and fail less badly."
    }
  },
  "models": [
    {
      "arm": "eg1",
      "name": "EG-1",
      "vendor": "Envious Labs",
      "product": "EnviousWispr",
      "parameters": "4B",
      "base_model": "Qwen3-4B-Instruct-2507",
      "training": "QLoRA fine-tune, adapter merged into the base weights",
      "quantisation": "Q5_K_M",
      "runtime": "llama.cpp",
      "size_on_disk_gb": 2.7,
      "memory_loaded_gb": 4.0,
      "licence": "EG-1 Community Model License. The model is free to use in the app and is NOT open source; the EnviousWispr app itself is GPLv3.",
      "prompt_source": "Its own training prompt, held by the publisher.",
      "note": "The version measured is the one shipping to users as of this run, installed on the test machine by the app. An older version is included as eg1_v1_superseded for reference and should not be quoted as current."
    },
    {
      "arm": "s1mini_lists",
      "name": "S1-mini",
      "vendor": "Superwhisper",
      "product": "Superwhisper",
      "parameters": "0.6B",
      "base_model": "Qwen3-0.6B",
      "quantisation": "Q4_K_M",
      "runtime": "llama.cpp",
      "size_on_disk_gb": 0.46,
      "memory_loaded_gb": 1.5,
      "licence": "Apache 2.0 plus a naming condition: it must be credited as \"S1-mini\" by \"Superwhisper\" wherever it is used. Open weights.",
      "source": "https://huggingface.co/superwhisper/s1-mini",
      "prompt_source": "Published on the model card: its own system prompt plus a control line setting register, structure and context.",
      "settings_used": "Styling: semi-formal, which the card names as its default and uses for its own published examples. Structure: lists for the quoted arm, prose reported separately.",
      "note": "Structure is a USER SETTING for this model, not a judgement it makes. Told to build lists it emitted one on 104 of 114 list cases; told prose it emitted none and scored 0% on that shape. Both arms are reported because neither alone describes the model.",
      "language": "English only"
    },
    {
      "arm": "fluid1_reasoning",
      "name": "Fluid-1",
      "vendor": "Altic",
      "product": "FluidVoice",
      "parameters": "4B class",
      "base_model": "Gemma 4 (E4B family)",
      "quantisation": "NVFP4, 4-bit",
      "runtime": "MLX",
      "size_on_disk_gb": 3.3,
      "memory_loaded_gb": 3.6,
      "licence": "Personal, non-commercial use, and the current checkpoints are restricted to use through official FluidVoice applications. Not redistributable.",
      "prompt_source": "NOT PUBLIC for this model. The instructions FluidVoice pairs with it live in a closed part of the app. This run used the dictation prompt published in their own open-source repository, assembled into the message the way their code assembles it.",
      "note": "Fluid-1 was tested with substituted instructions because its production instructions were unavailable. The effect of that substitution is unknown; these results do not establish performance in the official app. The official custom decoder and draft model were not replicated."
    }
  ],
  "results": {
    "eg1": {
      "label": "EG-1",
      "cases_scored": 1462,
      "pass_pct": 90.3,
      "serious_errors": 66,
      "judge_self_disagreement_pp": 0.3,
      "failure_types": {
        "entity_mutation": 37,
        "dropped_content": 30,
        "wrong_format": 14,
        "invented_content": 6
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 96.1,
          "serious_errors": 8
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 77.6,
          "serious_errors": 22
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 91.0,
          "serious_errors": 9
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 91.8,
          "serious_errors": 6
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 91.7,
          "serious_errors": 3
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 85.1,
          "serious_errors": 7
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 94.1,
          "serious_errors": 5
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 92.0,
          "serious_errors": 0
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 94.6,
          "serious_errors": 3
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 97.3,
          "serious_errors": 2
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 80.0,
          "serious_errors": 1
        }
      },
      "latency_single_stream_ms": {
        "p50": 312,
        "p95": 606,
        "mean": 341,
        "cases": 200
      }
    },
    "s1mini_lists": {
      "label": "S1-mini (lists)",
      "cases_scored": 1462,
      "pass_pct": 86.5,
      "serious_errors": 64,
      "judge_self_disagreement_pp": 3.1,
      "failure_types": {
        "dropped_content": 52,
        "entity_mutation": 31,
        "wrong_format": 31,
        "invented_content": 4,
        "under_polish": 1
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 93.9,
          "serious_errors": 14
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 59.8,
          "serious_errors": 24
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 95.0,
          "serious_errors": 3
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 92.5,
          "serious_errors": 6
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 91.7,
          "serious_errors": 3
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 82.5,
          "serious_errors": 9
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 93.1,
          "serious_errors": 2
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 72.4,
          "serious_errors": 1
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 98.6,
          "serious_errors": 0
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 100.0,
          "serious_errors": 0
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 70.0,
          "serious_errors": 2
        }
      },
      "latency_single_stream_ms": {
        "p50": 87,
        "p95": 176,
        "mean": 100,
        "cases": 200
      }
    },
    "s1mini_prose": {
      "label": "S1-mini (prose)",
      "cases_scored": 1462,
      "pass_pct": 82.0,
      "serious_errors": 60,
      "judge_self_disagreement_pp": 0.8,
      "failure_types": {
        "wrong_format": 113,
        "dropped_content": 41,
        "entity_mutation": 31,
        "invented_content": 3
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 95.5,
          "serious_errors": 11
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 58.4,
          "serious_errors": 22
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 95.0,
          "serious_errors": 6
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 92.5,
          "serious_errors": 7
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 95.8,
          "serious_errors": 1
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 0.0,
          "serious_errors": 8
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 94.1,
          "serious_errors": 2
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 97.7,
          "serious_errors": 0
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 95.9,
          "serious_errors": 1
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 100.0,
          "serious_errors": 0
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 73.3,
          "serious_errors": 2
        }
      }
    },
    "fluid1_reasoning": {
      "label": "Fluid-1 (reasoning)",
      "cases_scored": 1462,
      "pass_pct": 82.9,
      "serious_errors": 81,
      "judge_self_disagreement_pp": 2.8,
      "failure_types": {
        "dropped_content": 74,
        "wrong_format": 48,
        "entity_mutation": 29,
        "invented_content": 5,
        "verbatim_execution": 1
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 93.5,
          "serious_errors": 14
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 51.1,
          "serious_errors": 21
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 94.0,
          "serious_errors": 8
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 86.6,
          "serious_errors": 9
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 93.3,
          "serious_errors": 1
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 54.4,
          "serious_errors": 4
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 94.1,
          "serious_errors": 2
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 97.7,
          "serious_errors": 1
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 95.9,
          "serious_errors": 2
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 93.2,
          "serious_errors": 5
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 43.3,
          "serious_errors": 14
        }
      },
      "latency_single_stream_ms": {
        "p50": 267,
        "p95": 1746,
        "mean": 397,
        "cases": 200
      }
    },
    "fluid1_direct": {
      "label": "Fluid-1 (direct)",
      "cases_scored": 1462,
      "pass_pct": 80.3,
      "serious_errors": 86,
      "judge_self_disagreement_pp": 2.2,
      "failure_types": {
        "dropped_content": 83,
        "wrong_format": 61,
        "entity_mutation": 25,
        "invented_content": 9
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 92.9,
          "serious_errors": 13
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 44.3,
          "serious_errors": 30
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 94.0,
          "serious_errors": 7
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 85.1,
          "serious_errors": 10
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 92.5,
          "serious_errors": 3
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 43.9,
          "serious_errors": 3
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 93.1,
          "serious_errors": 3
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 95.4,
          "serious_errors": 1
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 95.9,
          "serious_errors": 2
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 94.5,
          "serious_errors": 4
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 30.0,
          "serious_errors": 10
        }
      },
      "latency_single_stream_ms": {
        "p50": 279,
        "p95": 424,
        "mean": 293,
        "cases": 200
      }
    },
    "eg1_v1_superseded": {
      "label": "EG-1 v1 (superseded)",
      "cases_scored": 1462,
      "pass_pct": 81.8,
      "serious_errors": 73,
      "judge_self_disagreement_pp": 1.5,
      "failure_types": {
        "wrong_format": 121,
        "dropped_content": 42,
        "entity_mutation": 37,
        "invented_content": 10
      },
      "per_shape": {
        "clean": {
          "cases": 309,
          "pass_pct": 95.8,
          "serious_errors": 10
        },
        "self_correction": {
          "cases": 219,
          "pass_pct": 70.3,
          "serious_errors": 16
        },
        "fillers_only": {
          "cases": 200,
          "pass_pct": 88.5,
          "serious_errors": 16
        },
        "voice_at_risk": {
          "cases": 134,
          "pass_pct": 90.3,
          "serious_errors": 8
        },
        "unfinished": {
          "cases": 120,
          "pass_pct": 90.0,
          "serious_errors": 3
        },
        "spoken_list": {
          "cases": 114,
          "pass_pct": 0.9,
          "serious_errors": 9
        },
        "topic_shift": {
          "cases": 102,
          "pass_pct": 94.1,
          "serious_errors": 3
        },
        "inline_enumeration": {
          "cases": 87,
          "pass_pct": 88.5,
          "serious_errors": 1
        },
        "connected_prose": {
          "cases": 74,
          "pass_pct": 91.9,
          "serious_errors": 5
        },
        "numbers_dates": {
          "cases": 73,
          "pass_pct": 98.6,
          "serious_errors": 1
        },
        "quoted_instruction": {
          "cases": 30,
          "pass_pct": 86.7,
          "serious_errors": 1
        }
      }
    }
  },
  "limitations": [
    "English only. Nothing here describes behaviour in other languages.",
    "The exam is the publisher's own and reflects its idea of what dictation looks like. Independent answer-key authors and a shared grading rubric address scoring consistency, not case selection. Model-identity masking is unverified.",
    "Fluid-1 was tested with substituted instructions because its production instructions were unavailable. The effect is unknown; these results do not establish its performance in the official app.",
    "Grading is by model judgement, which disagrees with itself by up to 3.1 points on this corpus. Gaps smaller than that are not readable.",
    "Latency includes each vendor's runtime, and one vendor's production decoder is not replicated. Read speed as indicative, not as a controlled measurement of the weights alone.",
    "Single machine, single run per arm. No variance across machines or repeated runs is reported."
  ],
  "source_note": "Adapted from the August 26, 2026 archived benchmark record and checked against the per-arm score summaries. Narrative claims that were not reconciled with run metadata have been qualified."
}