{"schema":"ime-site-bundle/1","built":"2026-11-19","about":"Data of the keyboard-state explorer. Built by the site's make_bundle.py; every value is traced in data/trace.tsv. Schema: data/SCHEMA.md.","status_legend":{"pilot":"Pilot, exploratory. Each number shows the items and calls behind it.","interim":"Main run, interim: taken while the main run was still going; numbers can change.","final":"Main run, final: the frozen analysis on complete data."},"sources":[{"id":"layout","path":"results/pilot_layout_20261115/SUMMARY.json","sha256_16":"22d4c3085f0b4c56"},{"id":"layout_floor","path":"results/pilot_layout_20261115/overlap_baseline.json","sha256_16":"5e9b79fcb7fab016"},{"id":"layout_settings","path":"results/pilot_layout_20261115/pilot.py","sha256_16":"05822fbde71cf416"},{"id":"layout_cost","path":"pilot cost table (PILOT_COST_PER_CALL.md)","sha256_16":"7ce1db04f4211390"},{"id":"zh","path":"results/pilot_zh_graded_20261116/SUMMARY.json","sha256_16":"1e8a6f87bb07aea6"},{"id":"fu","path":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","sha256_16":"027d727da19cd444"},{"id":"fu_posthoc","path":"results/pilot_zh_followup_20261116/POSTHOC_SUMMARY.json","sha256_16":"8ecd11a128c9d1e7"},{"id":"matrix","path":"docs/MODEL_MATRIX.md","sha256_16":"4d87a1f67ff194c3"},{"id":"roster","path":"registry smoke check (smoke_table.json)","sha256_16":"8202d9c941b63d44"},{"id":"variants","path":"data/belebele/variants_summary.json","sha256_16":"4e0724e7dad59a64"},{"id":"langnames","path":"site/data/langnames.json","sha256_16":"649a83a180f867ab"},{"id":"pack","path":"poster/v2/content/data.json","sha256_16":"d7b288328cc709bc"},{"id":"copy","path":"poster/v2/content/copy.json","sha256_16":"92829c310bef5eb2"},{"id":"zh_items","path":"results/pilot_zh_graded_20261116/items/tasks.jsonl","sha256_16":"0c2c7ded4cd1d4d7"},{"id":"wave0","path":"run plan of 19 Nov 2026 (WAVE0_DECISIONS.md)","sha256_16":"88810da315f6cee7"},{"id":"layout_md","path":"results/pilot_layout_20261115/SUMMARY.md","sha256_16":"f29d528c3f372d70"},{"id":"main_variants","path":"data/belebele/main_20261119/variants_main.summary.json","sha256_16":"2ddbe7c476e28bdc"},{"id":"breadth_variants","path":"data/belebele/main_20261119/variants_breadth.summary.json","sha256_16":"37ff27f79b2e39c7"},{"id":"map_variants","path":"data/belebele/main_20261119/variants_map.summary.json","sha256_16":"cbc4739a7090bfb3"}],"methods":{"wilson":"Wilson 95% interval over the calls (the layout pilot's own interval for a rate; it treats calls as independent). Computed by make_bundle.py from the counts.","wilson_maps":"Wilson 95% interval over the 150 calls of three random maps x 50 items (treats calls as independent, although each item is asked three times). Computed by make_bundle.py.","boot_item":"95% item-cluster bootstrap of the paired difference, 2,000 draws (layout pilot).","boot_item_maps":"95% item-cluster bootstrap of the per-item difference, real layout minus the mean of the three maps (layout pilot).","boot_class":"95% percentile bootstrap over the 18 classes, the settings held fixed: it describes these settings, not models in general (Chinese pilot and follow-up). Where all classes agree it has zero width; it says nothing about call-to-call variation.","wilson_calls":"Wilson 95% interval over this setting's calls, computed by make_bundle.py. It treats the three calls per item and the two items of a class as independent, so it is narrower than an interval over classes."},"operators":{"pilot_v2":"Pilot letter tables (operator version 2), not the main run's exact keystroke operator D3x. The Russian round trip is exact for 280 of 300 texts; Latin letters already inside the text become ambiguous.","d3x":"Exact keystroke operator D3x (layout_keys version 5, xkeyboard-config 2.49): every character is replaced by what the keys that type it on the intended layout give on the active one. Every keystroke is kept; characters on keys that print nothing on the active layout are lost."},"runs":[{"id":"pilot_layout_20261115","label":"Layout pilot","date":"2026-11-15","status":"pilot","what":"Wrong keyboard layouts on Belebele reading questions: 21 settings of 13 models, 10 language variants, 50 items each, one call per cell (Claude settings: the first 20 items of each cross-script layout); the six cross-script layouts are the core.","calls":44250,"items_per_language":50,"calls_per_cell":1,"source":"results/pilot_layout_20261115/SUMMARY.json"},{"id":"pilot_zh_graded_20261116","label":"Chinese pilot","date":"2026-11-16","status":"pilot","what":"96 constructed Chinese word items (not Belebele) in characters, toned and toneless pinyin; four reply formats; 21 settings, one call per cell. Items not yet checked by an independent native speaker.","calls":36288,"calls_per_cell":1,"source":"results/pilot_zh_graded_20261116/SUMMARY.json"},{"id":"pilot_zh_followup_20261116","label":"Chinese follow-up","date":"2026-11-16","status":"pilot","what":"The same items in toneless pinyin, three calls per undetermined cell, a note and a control sentence, four contrasts fixed in advance (a plan fixed before the first call, not a public pre-registration).","calls":35280,"calls_per_cell":3,"source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json"}],"numbers":{"layout.settings":{"value":21,"text":"21","source":"results/pilot_layout_20261115/SUMMARY.json","key":"coverage.core_ok","rule":"number of keys"},"layout.models":{"value":13,"text":"13","source":"results/pilot_layout_20261115/SUMMARY.json","key":"coverage.core_ok","rule":"distinct model ids (pilot.py)"},"layout.items":{"value":50,"text":"50","source":"results/pilot_layout_20261115/SUMMARY.json","key":"by_variant.terra_none.native.rus_Cyrl[1]","rule":null},"layout.cells":{"value":300,"text":"300","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.terra_none.cross.n","rule":null},"layout.variants":{"value":6,"text":"6","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.terra_none.cross.n / 50","rule":null},"layout.calls":{"value":44250,"text":"44,250","source":"results/pilot_layout_20261115/SUMMARY.json","key":"calls.records","rule":null},"layout.floor":{"value":34.8,"text":"34.8%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"overlap_floor_cross_layout","rule":"x100, 1 decimal"},"layout.floor.lo":{"value":29.2,"text":"29%","source":"results/pilot_layout_20261115/overlap_baseline.json","key":"min over the six variants of ['<variant>|kb_layout'][0]","rule":"x100, whole %"},"layout.floor.hi":{"value":41.0,"text":"41%","source":"results/pilot_layout_20261115/overlap_baseline.json","key":"max over the six variants of ['<variant>|kb_layout'][0]","rule":"x100, whole %"},"layout.floor.native":{"value":32.0,"text":"32.0%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"overlap_floor_cross_native","rule":"x100, 1 decimal"},"layout.chance":{"value":25.0,"text":"25%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"four options per item","rule":"by construction"},"layout.off.n":{"value":8,"text":"8","source":"results/pilot_layout_20261115/pilot.py","key":"SETTINGS[*].reasoning == 'off', less claude-haiku-5-5 (not plotted)","rule":null},"layout.off.atfloor":{"value":7,"text":"7","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.<setting>.cross.layout","rule":"reasoning-off settings within 10 points of the floor"},"layout.strong.n":{"value":4,"text":"4","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.<setting>.cross","rule":"settings within 10 points of native"},"layout.strong.gap_lo":{"value":4.0,"text":"4","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.<setting>.cross","rule":"native - layout, points"},"layout.strong.gap_hi":{"value":9.0,"text":"9","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.<setting>.cross","rule":"native - layout, points"},"layout.recover.n":{"value":8,"text":"8","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross.<setting>.cross.layout","rule":"settings that reason and end more than 15 points above the floor"},"layout.tokratio.lo":{"value":21.0,"text":"21","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross_outcomes.<setting>.reasoning_layout_mean / reasoning_native_mean","rule":"min over the recovering settings"},"layout.tokratio.hi":{"value":67.1,"text":"67","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross_outcomes.<setting>.reasoning_layout_mean / reasoning_native_mean","rule":"max over the recovering settings"},"layout.named.terra_none.plain":{"value":57.0,"text":"57.0%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"named[setting=terra_none,set=cross-script].plain","rule":"x100"},"layout.named.terra_none.named":{"value":86.7,"text":"86.7%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"named[setting=terra_none,set=cross-script].named","rule":"x100"},"layout.named.off_null":{"value":6,"text":"6","source":"results/pilot_layout_20261115/SUMMARY.json","key":"named[set=cross-script, reasoning off].lo, .hi","rule":"intervals that contain 0, terra_none excluded"},"layout.named.terra_two_lo":{"value":27.7,"text":"28","source":"results/pilot_layout_20261115/SUMMARY.json","key":"named / named_side_by_side [terra_none, cross-script].diff","rule":"smaller of the two runs, points"},"layout.named.terra_two_hi":{"value":29.7,"text":"30","source":"results/pilot_layout_20261115/SUMMARY.json","key":"named / named_side_by_side [terra_none, cross-script].diff","rule":"larger of the two runs, points"},"layout.recall.terra_none.real":{"value":94.0,"text":"94.0%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[setting=terra_none,lang=rus_Cyrl].real","rule":"x100"},"layout.recall.terra_none.random":{"value":40.7,"text":"40.7%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[setting=terra_none,lang=rus_Cyrl].random_mean","rule":"x100"},"layout.given.terra_none.table":{"value":88.0,"text":"88.0%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"table[setting=terra_none,lang=rus_Cyrl].table","rule":"x100"},"layout.apply.terra_none.gap":{"value":6.0,"text":"6","source":"results/pilot_layout_20261115/SUMMARY.json","key":"table[setting=terra_none,lang=rus_Cyrl].native - .table","rule":"points"},"layout.decipher.n":{"value":4,"text":"4","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[lang=rus_Cyrl, reasoning on].random_mean","rule":"settings that reason and read the random maps at 75% or more"},"layout.decipher.lo":{"value":76.0,"text":"76%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[lang=rus_Cyrl].random_mean","rule":"min over those settings"},"layout.decipher.hi":{"value":87.3,"text":"87%","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[lang=rus_Cyrl].random_mean","rule":"max over those settings"},"layout.decipher.ratio_lo":{"value":9.7,"text":"10","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[lang=rus_Cyrl].reasoning_random / reasoning_real","rule":"min, whole number ('about')"},"layout.decipher.ratio_hi":{"value":24.2,"text":"24","source":"results/pilot_layout_20261115/SUMMARY.json","key":"randcls[lang=rus_Cyrl].reasoning_random / reasoning_real","rule":"max, whole number"},"layout.trunc.dsflash_on":{"value":164,"text":"164","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross_outcomes.dsflash_on.outcomes.truncated","rule":null},"layout.trunc.kimi_on":{"value":45,"text":"45","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross_outcomes.kimi_on.outcomes.truncated","rule":null},"layout.trunc.glmflash_offreq":{"value":43,"text":"43","source":"results/pilot_layout_20261115/SUMMARY.json","key":"cross_outcomes.glmflash_offreq.outcomes.truncated","rule":null},"layout.calls_per_cell":{"value":1,"text":"one","source":"results/pilot_layout_20261115/pilot.py","key":"one call per cell (design document 9.1)","rule":null},"layout.roundtrip.k":{"value":280,"text":"280","source":"poster/v2/content/data.json","key":"numbers['layout.roundtrip.k']","rule":"poster pack; its source: design document: section 9.3 (not in a summary file; not recomputed here)"},"layout.roundtrip.n":{"value":300,"text":"300","source":"poster/v2/content/data.json","key":"numbers['layout.roundtrip.n']","rule":"poster pack; its source: design document: section 9.3"},"zh.classes":{"value":18,"text":"18","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"hard_pairs.classes","rule":null},"zh.u_items":{"value":36,"text":"36","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"2 x hard_pairs.classes","rule":null},"zh.settings":{"value":21,"text":"21","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"settings","rule":"number of settings"},"site.zh.items":{"value":96,"text":"96","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"n_items","rule":"sum over the item groups"},"zh.bound":{"value":50.0,"text":"50%","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"hard_pairs.expected_accuracy_bound_pct.keyboard_clean","rule":null},"zh.m":{"value":2,"text":"two","source":"poster/v2/content/data.json","key":"numbers['zh.m']","rule":"poster pack; its source: by construction (design document section 2, K2: m = 2)"},"zh.zero":{"value":0,"text":"0","source":"poster/v2/content/data.json","key":"numbers['zh.zero']","rule":"poster pack; its source: design document: section 10.1, reply format (2)"},"date.layout":{"value":null,"text":"15 Nov 2026","source":"poster/v2/content/data.json","key":"numbers['date.layout']","rule":"poster pack; its source: design document: heading of section 9 ('Pilot of 15 Nov 2026')"},"date.zh":{"value":null,"text":"16 Nov 2026","source":"poster/v2/content/data.json","key":"numbers['date.zh']","rule":"poster pack; its source: design document: heading of section 10 ('Pilot of 16 Nov 2026') and of section 10.7"},"zh.fc3.pct":{"value":48.9,"text":"48.9%","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"k2.pooled.pct","rule":null},"zh.fc3.ci":{"value":[46.7,50.6],"text":"[46.7, 50.6]","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"k2.pooled.ci","rule":null},"zh.fc3.calls":{"value":2268,"text":"2,268","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"k2.pooled.n","rule":null},"zh.fc3.per_cell":{"value":3,"text":"three","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"k2.pooled.n / (36 items x 21 settings)","rule":null},"zh.pilot.calls_per_cell":{"value":1,"text":"one","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"n_call_lines = 21 settings x 1,728 cells","rule":null},"zh.rescue.pct":{"value":99.1,"text":"99.1%","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"pooled.rows.all['forced choice, toneless pinyin, rescued']","rule":"k/n x100"},"zh.rescue.minority.pct":{"value":98.1,"text":"98.1%","source":"results/pilot_zh_graded_20261116/SUMMARY.json","key":"rev.c1c2.c2.rates.minority.kn","rule":"k/n x100"},"zh.flag.open.none":{"value":52.6,"text":"52.6%","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"arms.open.U_says.pct","rule":null},"zh.flag.free.none":{"value":21.6,"text":"21.6%","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"arms.free.U_says.pct","rule":null},"zh.bare.free":{"value":60.6,"text":"60.6%","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"arms.free.U_bare.pct","rule":null},"zh.note_gain.lo":{"value":4.9,"text":"4.9","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"min of primary.H2a/H2b/H2c.est","rule":null},"zh.note_gain.hi":{"value":12.9,"text":"12.9","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"max of primary.H2a/H2b/H2c.est","rule":null},"zh.ci_level":{"value":95,"text":"95%","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"level","rule":"x100"},"site.zh.h1.est":{"value":31.0,"text":"31.0","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"primary.H1.est","rule":"points, 1 decimal"},"site.zh.h1.form":{"value":29.2,"text":"29.2","source":"results/pilot_zh_followup_20261116/POSTHOC_SUMMARY.json","key":"form_or_rate.H1.form","rule":"points of H1 that go with the change in reply form, 1 decimal"},"site.zh.above_bound":{"value":3,"text":"3","source":"results/pilot_zh_followup_20261116/FOLLOWUP_SUMMARY.json","key":"k2.per_setting.<setting>.pct","rule":"settings whose own forced-choice share is above the bound"},"site.planned":{"value":145,"text":"145","source":"registry smoke check (smoke_table.json)","key":"<setting>.verdict","rule":"registered settings that passed the 5-call smoke check (PASS or PASS*)"},"model.gpt-6.1-terra":{"value":null,"text":"gpt-6.1-terra","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.gpt-6.1-sol":{"value":null,"text":"gpt-6.1-sol","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.gpt-6-astra":{"value":null,"text":"gpt-6-astra","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.claude-sonnet-5-5":{"value":null,"text":"claude-sonnet-5-5","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.kimi-k3-1-pro-261024":{"value":null,"text":"kimi-k3-1-pro-261024","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.deepseek-v4-1-flash-261017":{"value":null,"text":"deepseek-v4-1-flash-261017","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"model.glm-5-3-flash-260828":{"value":null,"text":"glm-5-3-flash-260828","source":"registry smoke check (smoke_table.json)","key":"<setting>.model_id","rule":null},"abs.calls":{"value":48,"text":"48","source":"poster/v2/content/data.json","key":"numbers['abs.calls']","rule":"poster pack; its source: reports/real_scores.jsonl and reports/real_scores_*.jsonl: records with quality_status == scored"},"site.settings":{"value":147,"text":"147","source":"registry smoke check (smoke_table.json)","key":"keys less the pilot's glm_5_3_flash=off-requested","rule":"count"},"site.models":{"value":50,"text":"50","source":"registry smoke check (smoke_table.json)","key":"distinct model keys of those settings","rule":"count"},"site.variants":{"value":122,"text":"122","source":"data/belebele/variants_summary.json","key":"n_language_variants","rule":null},"model.claude-haiku-5-5":{"value":null,"text":"claude-haiku-5-5","source":"poster/v2/content/data.json","key":"numbers['model.claude-haiku-5-5']","rule":"poster pack; its source: results/pilot_layout_20261115/pilot.py: SETTINGS[*].model"},"abs.one":{"value":1.0,"text":"1.00","source":"poster/v2/content/data.json","key":"numbers['abs.one']","rule":"poster pack; its source: reports/real_scores*.jsonl: mean quality of every row with condition native, and of hom_codex / hom_claude with condition mixed_keyboard_terms (computed; equal by assertion)"},"abs.routine.tasks":{"value":2,"text":"two","source":"poster/v2/content/data.json","key":"numbers['abs.routine.tasks']","rule":"poster pack; its source: reports/real_scores.jsonl and reports/real_scores_zh_cn_con_01_six_conditions.jsonl: distinct task_id"},"abs.hom.tasks":{"value":3,"text":"three","source":"poster/v2/content/data.json","key":"numbers['abs.hom.tasks']","rule":"poster pack; its source: reports/real_scores_phase4_homophone_stress.jsonl: distinct task_id"},"abs.hom_codex.keyboard_clean":{"value":0.333,"text":"0.333","source":"poster/v2/content/data.json","key":"numbers['abs.hom_codex.keyboard_clean']","rule":"poster pack; its source: reports/real_scores_phase4_homophone_stress.jsonl: mean quality over the 3 calls with condition keyboard_clean"},"date.abstract":{"value":null,"text":"May–June 2026","source":"poster/v2/content/data.json","key":"numbers['date.abstract']","rule":"poster pack; its source: reports/codex_*_report.md and claude_sonnet_homophone_small_report.md: **Date:** lines (2026-05-31 to 2026-06-08)"}},"examples":{"wrong_layout":{"intended":"привет","typed":"ghbdtn","gloss":"hello","intended_layout":"Russian ЙЦУКЕН","active_layout":"US English (QWERTY)","phrase_intended":"Привет, мир","phrase_typed":"Ghbdtn? vbh","phrase_gloss":"hello, world","pair":"ru>us"},"toneless_pinyin":{"typed":"beizi","words":[{"characters":"杯子","toned":"bēizi","gloss":"cup","optional":false},{"characters":"被子","toned":"bèizi","gloss":"quilt","optional":false},{"characters":"辈子","toned":"bèizi","gloss":"lifetime","optional":true}]},"pinyin_classes":[{"class_id":"beizi","typed":"beizi","level":3,"words":[{"word":"杯子","gloss":"cup","toned":"bēizi"},{"word":"被子","gloss":"quilt","toned":"bèizi"},{"word":"辈子","gloss":"lifetime","toned":null}],"items":["zh-CN-grad-L3-01","zh-CN-grad-L3-02"]},{"class_id":"daikuan","typed":"daikuan","level":2,"words":[{"word":"贷款","gloss":"loan (borrowed money)","toned":"dàikuǎn"},{"word":"带宽","gloss":"bandwidth","toned":"dàikuān"}],"items":["zh-CN-grad-L2-01","zh-CN-grad-L2-02"]},{"class_id":"shuimian","typed":"shuimian","level":2,"words":[{"word":"睡眠","gloss":"sleep","toned":"shuìmián"},{"word":"水面","gloss":"water surface","toned":"shuǐmiàn"}],"items":["zh-CN-grad-L2-09","zh-CN-grad-L2-10"]},{"class_id":"guomin","typed":"guomin","level":2,"words":[{"word":"国民","gloss":"citizens of a nation","toned":"guómín"},{"word":"过敏","gloss":"allergy","toned":"guòmǐn"}],"items":["zh-CN-grad-L2-03","zh-CN-grad-L2-04"]},{"class_id":"zifu","typed":"zifu","level":2,"words":[{"word":"自负","gloss":"conceited","toned":"zìfù"},{"word":"字符","gloss":"character (text symbol)","toned":"zìfú"}],"items":["zh-CN-grad-L2-11","zh-CN-grad-L2-12"]},{"class_id":"jianpan","typed":"jianpan","level":1,"words":[{"word":"键盘","gloss":"keyboard","toned":"jiànpán"}],"items":["zh-CN-grad-L1-02"]}],"pinyin_source":"From the Chinese pilot's item file; the items have not yet been checked by an independent native speaker."},"plan":{"source":"run plan of 19 Nov 2026","date":"2026-11-19","confirmatory":[{"id":"C1","label":"Full disclosure of the keyboard state","a":"kb_layout_keys_named","b":"kb_layout_keys_ctrl","what":"Accuracy with the layout named (b) minus accuracy with the control sentence (b'). Counted as supported only if it passes the Holm correction and (b) minus no sentence (a) is positive. Its two parts are secondary: recognition (b'') minus (b'), identification (b) minus (b'')."},{"id":"C2","label":"The standard layout against random permutations of the same keys","a":"kb_layout_keys","b":"kb_lk_rand","what":"Accuracy on the real wrong layout (a) minus accuracy on class-preserving random key maps of the same keys (d). A real layout may differ from a random one in more than exposure; the phonetic arm (c) is reported beside it."},{"id":"C3","label":"The information in the table","a":"kb_lk_rand_table","b":"kb_lk_rand_shamtable","what":"Random key map with its key table (f) minus the same map with a sham table of the same form and length (f0)."},{"id":"C4","label":"The cost moves to tokens","a":"kb_layout_keys","b":"native","what":"Pairs of one model with reasoning off and on; the list of pairs is fixed at the freeze, after a protocol check on pilot items. In every pair the accuracy loss on wrong-layout text is smaller with reasoning on, and with reasoning on the model spends more reasoning tokens than on the text as written."},{"id":"C5","label":"Prompted abstention under the explicit format","a":"note","b":"control","what":"Mandarin items whose typed text does not settle the answer, with a 'cannot tell' answer offered: share of replies that choose it after the disclosure sentence minus after the control sentence."}],"correction":"Holm over the five","alpha":0.05,"ladder_languages":["rus_Cyrl","heb_Hebr","hye_Armn","arb_Arab"],"items":{"first_rank":50,"last_rank":149,"extended":[150,249],"pilot_ranks":"0 to 49","text":"C1 to C4: Belebele items of draw ranks 50 to 149, the same for every setting (items whose passage also occurs among ranks 0 to 49 are left out); ranks 150 to 249 where the budget allows, as a secondary extended-set estimate. C5: the Mandarin items built at the bound. The pilots used ranks 0 to 49."},"cluster":"the Belebele passage for C1 to C4, the same-spelling class for C5","scope":"Four cross-script layouts under an English instruction for C1 to C4 (Russian, Hebrew, Armenian, Arabic); Mandarin for C5. Everything else is descriptive.","freeze":"The plan, the items and the analysis code are frozen and time-stamped before the first main-run call. A public registration follows later.","exploratory":["The roster: every registered setting that passed the 5-call smoke check, on fewer items and arms, for this explorer.","The map: all 122 Belebele variants as written and romanised or without diacritics, on five settings (gpt-6-luna effort none and high, deepseek-v4-1-flash thinking off, doubao-seed-2-1-lite thinking off, claude-haiku-5-5 thinking off).","Breadth: every language with a wrong-layout condition (29), and English typed on four other layouts.","The arm with another map's table (f x), on some settings only.","Japanese romaji and Vietnamese VNI keystrokes beside the Chinese and Vietnamese arms.","A secondary family on mechanism (spaced text, a generic hint, three examples, reversed options), never part of the five tests."]},"design":{"built":{"acm_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"afr_Latn":["kb_clean","native"],"als_Latn":["kb_clean","native"],"amh_Ethi":["kb_clean","native"],"apc_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"arb_Arab":["kb_clean","kb_clean@question_only","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_ctrl","kb_layout_keys_ctrl@question_only","kb_layout_keys_named","kb_layout_keys_named@question_only","kb_layout_keys_recog","kb_layout_keys_recog@question_only","kb_lk_rand","kb_lk_rand@question_only","kb_lk_rand_note","kb_lk_rand_note@question_only","kb_lk_rand_shamtable","kb_lk_rand_shamtable@question_only","kb_lk_rand_table","kb_lk_rand_table@question_only","kb_lk_rand_xtable","kb_lk_rand_xtable@question_only","native","native_ctrl","native_nopassage"],"arb_Latn":["kb_clean","native"],"ars_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"ary_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"arz_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","native"],"asm_Beng":["kb_clean","native"],"azj_Latn":["kb_clean","native"],"bam_Latn":["kb_clean","native"],"ben_Beng":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"ben_Latn":["kb_clean","native"],"bod_Tibt":["kb_clean","native"],"bul_Cyrl":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","native"],"cat_Latn":["kb_clean","native"],"ceb_Latn":["kb_clean","native"],"ces_Latn":["kb_clean","native"],"ckb_Arab":["kb_clean","native"],"dan_Latn":["kb_clean","native"],"deu_Latn":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"ell_Grek":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"eng_Latn":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","kb_layout_keys_rev_ara","kb_layout_keys_rev_il","kb_layout_keys_rev_ko","kb_layout_keys_rev_ru","native"],"est_Latn":["kb_clean","native"],"eus_Latn":["kb_clean","native"],"fin_Latn":["kb_clean","native"],"fra_Latn":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"fuv_Latn":["kb_clean","native"],"gaz_Latn":["kb_clean","native"],"grn_Latn":["kb_clean","native"],"guj_Gujr":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"hat_Latn":["kb_clean","native"],"hau_Latn":["kb_clean","native"],"heb_Hebr":["kb_clean","kb_clean@question_only","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_ctrl","kb_layout_keys_ctrl@question_only","kb_layout_keys_named","kb_layout_keys_named@question_only","kb_layout_keys_recog","kb_layout_keys_recog@question_only","kb_lk_phonetic","kb_lk_phonetic@question_only","kb_lk_rand","kb_lk_rand@question_only","kb_lk_rand_note","kb_lk_rand_note@question_only","kb_lk_rand_shamtable","kb_lk_rand_shamtable@question_only","kb_lk_rand_table","kb_lk_rand_table@question_only","kb_lk_rand_xtable","kb_lk_rand_xtable@question_only","native","native_ctrl","native_nopassage"],"hin_Deva":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"hin_Latn":["kb_clean","native"],"hrv_Latn":["kb_clean","native"],"hun_Latn":["kb_clean","native"],"hye_Armn":["kb_clean","kb_clean@question_only","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_ctrl","kb_layout_keys_ctrl@question_only","kb_layout_keys_named","kb_layout_keys_named@question_only","kb_layout_keys_recog","kb_layout_keys_recog@question_only","kb_lk_phonetic","kb_lk_phonetic@question_only","kb_lk_rand","kb_lk_rand@question_only","kb_lk_rand_note","kb_lk_rand_note@question_only","kb_lk_rand_shamtable","kb_lk_rand_shamtable@question_only","kb_lk_rand_table","kb_lk_rand_table@question_only","kb_lk_rand_xtable","kb_lk_rand_xtable@question_only","native","native_ctrl","native_nopassage"],"ibo_Latn":["kb_clean","native"],"ilo_Latn":["kb_clean","native"],"ind_Latn":["kb_clean","native"],"isl_Latn":["kb_clean","native"],"ita_Latn":["kb_clean","native"],"jav_Latn":["kb_clean","native"],"jpn_Jpan":["kb_clean","kb_romaji_ime","kb_romaji_ime@question_only","native"],"kac_Latn":["kb_clean","native"],"kan_Knda":["kb_clean","native"],"kat_Geor":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"kaz_Cyrl":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"kea_Latn":["kb_clean","native"],"khk_Cyrl":["kb_clean","native"],"khm_Khmr":["kb_clean","native"],"kin_Latn":["kb_clean","native"],"kir_Cyrl":["kb_clean","native"],"kor_Hang":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","native"],"lao_Laoo":["kb_clean","native"],"lin_Latn":["kb_clean","native"],"lit_Latn":["kb_clean","native"],"lug_Latn":["kb_clean","native"],"luo_Latn":["kb_clean","native"],"lvs_Latn":["kb_clean","native"],"mal_Mlym":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"mar_Deva":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"mkd_Cyrl":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"mlt_Latn":["kb_clean","native"],"mri_Latn":["kb_clean","native"],"mya_Mymr":["kb_clean","native"],"nld_Latn":["kb_clean","native"],"nob_Latn":["kb_clean","native"],"npi_Deva":["kb_clean","native"],"npi_Latn":["kb_clean","native"],"nso_Latn":["kb_clean","native"],"nya_Latn":["kb_clean","native"],"ory_Orya":["kb_clean","native"],"pan_Guru":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"pbt_Arab":["kb_clean","native"],"pes_Arab":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"plt_Latn":["kb_clean","native"],"pol_Latn":["kb_clean","native"],"por_Latn":["kb_clean","native"],"ron_Latn":["kb_clean","native"],"rus_Cyrl":["kb_clean","kb_clean@question_only","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_ctrl","kb_layout_keys_ctrl@question_only","kb_layout_keys_named","kb_layout_keys_named@question_only","kb_layout_keys_recog","kb_layout_keys_recog@question_only","kb_lk_phonetic","kb_lk_phonetic@question_only","kb_lk_rand","kb_lk_rand@question_only","kb_lk_rand_note","kb_lk_rand_note@question_only","kb_lk_rand_shamtable","kb_lk_rand_shamtable@question_only","kb_lk_rand_table","kb_lk_rand_table@question_only","kb_lk_rand_xtable","kb_lk_rand_xtable@question_only","native","native_ctrl","native_nopassage"],"shn_Mymr":["kb_clean","native"],"sin_Latn":["kb_clean","native"],"sin_Sinh":["kb_clean","native"],"slk_Latn":["kb_clean","native"],"slv_Latn":["kb_clean","native"],"sna_Latn":["kb_clean","native"],"snd_Arab":["kb_clean","native"],"som_Latn":["kb_clean","native"],"sot_Latn":["kb_clean","native"],"spa_Latn":["kb_clean","native"],"srp_Cyrl":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"ssw_Latn":["kb_clean","native"],"sun_Latn":["kb_clean","native"],"swe_Latn":["kb_clean","native"],"swh_Latn":["kb_clean","native"],"tam_Taml":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","native"],"tel_Telu":["kb_clean","native"],"tgk_Cyrl":["kb_clean","native"],"tgl_Latn":["kb_clean","native"],"tha_Thai":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","native"],"tir_Ethi":["kb_clean","native"],"tsn_Latn":["kb_clean","native"],"tso_Latn":["kb_clean","native"],"tur_Latn":["kb_clean","native"],"ukr_Cyrl":["kb_clean","kb_layout_keys","kb_layout_keys@question_only","kb_layout_keys_named","native"],"urd_Arab":["kb_clean","native"],"urd_Latn":["kb_clean","native"],"uzn_Latn":["kb_clean","native"],"vie_Latn":["kb_clean","kb_clean@question_only","kb_telex","kb_telex@question_only","kb_vni","kb_vni@question_only","native"],"war_Latn":["kb_clean","native"],"wol_Latn":["kb_clean","native"],"xho_Latn":["kb_clean","native"],"yor_Latn":["kb_clean","native"],"zho_Hans":["kb_clean","kb_pinyin_initials","kb_pinyin_initials@question_only","kb_pinyin_tone","kb_pinyin_tone@question_only","kb_pinyin_unspaced","kb_pinyin_unspaced@question_only","kb_pinyin_word","kb_pinyin_word@question_only","native"],"zho_Hant":["kb_clean","native"],"zsm_Latn":["kb_clean","native"],"zul_Latn":["kb_clean","native"]},"map_settings":["gpt_6_luna=off","gpt_6_luna=high","deepseek_v4_1_flash=off","doubao_seed_2_1_lite=off","claude_haiku_5_5=off"],"not_planned":[{"model_key":"claude_haiku_5_5","arms_kind":"lossless","why":"the run plan keeps claude-haiku-5-5 out of wrong-layout arms"}],"roundtrip_main":{"rus_Cyrl":[196,200],"heb_Hebr":[198,200],"hye_Armn":[178,200],"arb_Arab":[186,200]},"roundtrip_ranks":[50,250],"pilot_pooled_only":[{"setting":"gpt_6_luna=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"gpt_6_luna=high","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"gpt_6_1_terra=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"gpt_6_1_terra=high","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"gpt_6_1_sol=low","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"deepseek_v4_1_flash=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"deepseek_v4_pro=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"},{"setting":"kimi_k3_1_pro=off","arm":"kb_layout_keys_named","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"pooled":"pool_cross6","run":"pilot_layout_20261115"}]},"layout_families":{"cross-script":{"label":"Cross-script layouts","note":"Typed with the US layout active: key positions have nothing to do with how the letters sound.","langs":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"]},"greek":{"label":"Greek: a phonetic layout","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","langs":["ell_Grek"]},"near-identity":{"label":"German and French: near-identity layouts","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","langs":["deu_Latn","fra_Latn"]},"shift":{"label":"English: keys shifted right (a cipher)","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","langs":["eng_Latn"]}},"copy":{"headline":{"text":"Garbled text can be decoded, at a cost. Ambiguous text cannot."},"deck":{"text":"Keyboard-state input is text as typed, before or without conversion by an input method editor (IME). We tested {{layout.models}} models in {{layout.settings}} settings (reasoning off or on) on four-option questions typed this way."},"callout_floor":{"text":"The floor without reading: what letter overlap alone scores on the wrong-layout text.","big":"{{layout.floor}}"},"explain_wrong":{"text":"Garbled. Russian typed on the wrong keyboard layout (English). Every keystroke survives, so undoing the key map gives the text back.","emphasis":"Garbled."},"explain_pinyin":{"text":"Ambiguous. A Chinese word typed alone as pinyin without tones, before the IME picks the characters. Here one spelling stands for more than one word, and nothing in the text says which.","emphasis":"Ambiguous."},"layout_head":{"text":"Garbled text: nothing is lost"},"layout_tag":{"text":"pilot, {{date.layout}} · exploratory · {{layout.variants}} cross-script layouts, {{layout.items}} items each, {{layout.calls_per_cell}} call per item","changed_for_site":"poster-only phrase dropped"},"layout_lead":{"text":"In every setting plotted, decoding has a cost: with reasoning on, reasoning tokens and, in part, accuracy; with it off, accuracy."},"callout_off":{"text":"settings with reasoning off score within ten points of the floor.","big":"{{layout.off.atfloor}} of {{layout.off.n}}"},"callout_tokens":{"text":"the reasoning tokens of native text: what the {{layout.recover.n}} settings that recover spend on the same items typed on the wrong layout.","big":"{{layout.tokratio.lo}} to {{layout.tokratio.hi}} times"},"callout_gap":{"text":"below native text: where the best {{layout.strong.n}} settings still end, with reasoning on.","big":"{{layout.strong.gap_lo}} to {{layout.strong.gap_hi}} points"},"cap_figA":{"text":"Each row: one setting (model, reasoning level), on native text and on the same items typed on the wrong layout."},"fn_floor":{"text":"Floor without reading: expected accuracy of always picking the option with the largest share of its letter triples found in the passage, ties sharing credit, computed on the wrong-layout text: {{layout.floor}} on average, {{layout.floor.lo}} to {{layout.floor.hi}} by language. Chance: {{layout.chance}}."},"fn_layouts":{"text":"Layouts: Russian, Korean, Hebrew, Thai, Arabic and Egyptian Arabic (the last two share one keyboard). Every reply counts; one cut off at the token limit counts as wrong. Reasoning tokens as each API reports them."},"fn_dagger":{"text":"† Many wrong-layout replies reached the output-token limit and count as wrong: {{layout.trunc.dsflash_on}} of {{layout.cells}} ({{model.deepseek-v4-1-flash-261017}}, thinking on), {{layout.trunc.kimi_on}} of {{layout.cells}} ({{model.kimi-k3-1-pro-261024}}, thinking on), {{layout.trunc.glmflash_offreq}} of {{layout.cells}} ({{model.glm-5-3-flash-260828}}, thinking off requested)."},"fn_notshown":{"text":"Not shown: {{model.claude-haiku-5-5}}, whose wrong-layout cells were not run in full."},"fn_offreq":{"text":"Listed under reasoning on: {{model.glm-5-3-flash-260828}}, which reports reasoning tokens although thinking off was requested."},"fn_roundtrip":{"text":"Round trip: undoing the pilot’s wrong-layout converter restores {{layout.roundtrip.k}} of {{layout.roundtrip.n}} Russian texts (passages, questions, options) exactly; Latin letters already inside Russian text become ambiguous."},"steps_lead":{"text":"Hypothesis: reading wrong-layout text takes up to four steps. Models fail at different steps (a description, not a tested hypothesis)."},"step_recognise":{"text":"Recognise that a key map applies, and which. One sentence naming the layout lifts {{model.gpt-6.1-terra}}, reasoning off, from {{layout.named.terra_none.plain}} to {{layout.named.terra_none.named}}; {{layout.named.off_null}} other reasoning-off settings show no clear gain (one wording, no control sentence).","big":"{{layout.named.terra_none.plain}} → {{layout.named.terra_none.named}}","emphasis":"Recognise that a key map applies, and which."},"step_recall":{"text":"Recall the map, or be given it. Same setting, Russian: {{layout.recall.terra_none.real}} on the real layout, as on native text; {{layout.recall.terra_none.random}} on equally reversible random maps; {{layout.given.terra_none.table}} on the first of them, given with a note and its letter table.","big":"{{layout.recall.terra_none.real}} · {{layout.recall.terra_none.random}} · {{layout.given.terra_none.table}}","emphasis":"Recall the map, or be given it."},"step_apply":{"text":"Apply it. Even given the table, the same setting stays {{layout.apply.terra_none.gap}} points below native text on the same {{layout.items}} items.","big":"{{layout.apply.terra_none.gap}} points","emphasis":"Apply it."},"step_decipher":{"text":"Decipher it, where no map is known. With reasoning, {{layout.decipher.n}} settings read Russian typed through random maps at {{layout.decipher.lo}} to {{layout.decipher.hi}}, for about {{layout.decipher.ratio_lo}} to {{layout.decipher.ratio_hi}} times their real-layout reasoning tokens.","big":"{{layout.decipher.lo}} to {{layout.decipher.hi}}","emphasis":"Decipher it, where no map is known."},"fn_decipher":{"text":"Decipher step: {{model.gpt-6.1-terra}} (effort high), {{model.gpt-6.1-sol}} (effort low and high) and {{model.gpt-6-astra}} (effort low)."},"pinyin_head":{"text":"Ambiguous text: something is lost"},"pinyin_tag":{"text":"pilot and follow-up, {{date.zh}} · exploratory · {{zh.classes}} same-spelling classes · {{zh.pilot.calls_per_cell}} call per item, {{zh.fc3.per_cell}} in the follow-up · items not yet native-checked · follow-up plan fixed in advance","changed_for_site":"poster-only phrase dropped"},"pinyin_lead":{"text":"In each of {{zh.classes}} classes of same-spelling words, {{zh.m}} questions send the model the same text, byte for byte, and need different answers. No reader can expect more than {{zh.bound}}."},"pinyin_bound":{"text":"Forced to pick an option, the {{zh.settings}} settings together score {{zh.fc3.pct}} {{zh.fc3.ci}}: at the bound. Other same-spelling words, in sentences written to allow one reading, score {{zh.rescue.pct}} in the pilot, and {{zh.rescue.minority.pct}} where the rarer word is the answer."},"pinyin_flag":{"text":"Where the text cannot settle the answer, the only better behaviour is to say so. In free text, {{zh.flag.open.none}} of calls do. When a closing line also asks for the option number, {{zh.flag.free.none}} do, and {{zh.bare.free}} of replies are a bare number."},"pinyin_note":{"text":"A note saying that the text is pinyin without tones matters less than the closing line: {{zh.note_gain.lo}} to {{zh.note_gain.hi}} points over a control sentence that names neither, unevenly across these settings."},"cap_zhBound":{"text":"Forced choice on the {{zh.u_items}} word-alone items, against the {{zh.bound}} bound. The sentence items use other words."},"cap_zhFlag":{"text":"Calls that answer {{zh.zero}} (the text does not decide) or say so, by reply format and by the sentence placed before the question. {{zh.fc3.calls}} calls per bar."},"fn_judges":{"text":"Free-text replies were labelled by model judges that are also models under test ({{model.claude-sonnet-5-5}} and {{model.gpt-6.1-sol}}, with {{model.kimi-k3-1-pro-261024}} on their splits); no person has checked a label."},"lab_ci":{"text":"Brackets: {{zh.ci_level}} interval over the classes, with these {{zh.settings}} settings held fixed."},"fn_sentences":{"text":"Note: “Note: the Chinese text below was typed as Hanyu Pinyin without tone marks.” Control sentence: “Note: the Chinese text below was typed in lower-case letters with spaces between words.”"},"rules_head":{"text":"For input pipelines: three suggestions, not yet tested"},"rule_state":{"text":"Send the keyboard state (layout, input mode) with the text, where the client knows it. Naming the layout gave {{model.gpt-6.1-terra}} {{layout.named.terra_two_lo}} to {{layout.named.terra_two_hi}} points in two runs, and {{layout.named.off_null}} other reasoning-off settings no clear gain.","emphasis":"Send the keyboard state (layout, input mode) with the text, where the client knows it."},"rule_collide":{"text":"Keep only same-spelling words in native script. In the earlier study this scored {{abs.one}}, but the kept words were the answers themselves.","emphasis":"Keep only same-spelling words in native script."},"rule_room":{"text":"Leave room to flag ambiguity. A reply that is a bare option number cannot flag it.","emphasis":"Leave room to flag ambiguity."},"limits_head":{"text":"Limits"},"limits":{"text":"Exploratory: {{layout.items}} items per layout, {{layout.calls_per_cell}} call per item; Chinese items not yet checked by an independent native speaker; model judges, no human labels; {{zh.settings}} settings chosen for convenience; one wording per sentence and reply format. Abstract: {{abs.calls}} valid calls, Chinese only.","short":"Exploratory: {{layout.items}} items per layout, {{layout.calls_per_cell}} call per item; Chinese items not yet native-checked; model judges; {{zh.settings}} convenience settings; one wording per sentence and format; abstract: {{abs.calls}} valid calls, Chinese only."},"next_head":{"text":"Next"},"next":{"text":"The planned main run: conditions that separate the four steps, items built at the bound in several languages, and notes tested against control sentences.","changed_for_site":"no public registration exists yet (run plan, section 7)"},"lab_native":{"text":"native text"},"lab_wrong":{"text":"wrong layout"},"lab_floor":{"text":"floor without reading"},"lab_chance":{"text":"chance"},"lab_reasoning_on":{"text":"reasoning on"},"lab_reasoning_off":{"text":"reasoning off"},"lab_typed":{"text":"typed"},"lab_meant":{"text":"meant"},"lab_one_call":{"text":"word alone, {{zh.pilot.calls_per_cell}} call per item"},"lab_three_calls":{"text":"word alone, {{zh.fc3.per_cell}} calls per item"},"lab_sentence":{"text":"other words, in a sentence"},"lab_fmt_ab":{"text":"answer {{zh.zero}} offered"},"lab_fmt_free":{"text":"free text, closing line asks for the option number"},"lab_fmt_open":{"text":"free text, no closing line"},"lab_arm_none":{"text":"no sentence"},"lab_arm_control":{"text":"control sentence"},"lab_arm_note":{"text":"note"},"lab_bound":{"text":"bound for any reader"},"title":{"text":"Skipping the IME? Keyboard-Friendly Multilingual Prompting for LLM Agents"},"authors":{"text":"Xiaoxiao Zhouyi and Yanzhen Li"},"affiliation":{"text":"School of Engineering Mathematics and Technology, University of Bristol"},"kicker":{"text":"UKAIRS 2026 · Emerging Research · Poster UKAIRS26-89"},"statement":{"text":"Use of AI tools: LLM coding and writing assistants were used to build the evaluation framework, to run and check the analyses, and to prepare the poster, including its text and layout. Free-text replies in the November pilots were labelled by model judges. Numbers are generated by scripts from the run records. The authors designed the study and take responsibility for all content.","changed_for_site":"web wording"},"abs_head":{"text":"Earlier study: coding agents, Chinese prompts","short":"Earlier study: coding agents"},"abs_body":{"text":"Codex CLI kept its score on {{abs.routine.tasks}} routine prompts typed as pinyin without tones. On {{abs.hom.tasks}} tasks whose answer is an exact term, both agents fell to {{abs.hom_codex.keyboard_clean}}, and scored {{abs.one}} with those terms kept in characters (one call per task)."},"abs_tag":{"text":"The accepted abstract · {{abs.calls}} valid calls · Codex CLI and Claude Code · {{date.abstract}}"}},"arms":[{"id":"native","label":"As written","short":"as written","kind":"native","told":false,"design":"core","what":"The passage, question and options as they are in Belebele: the text as meant."},{"id":"kb_layout_keys","label":"Wrong layout","short":"wrong layout","kind":"lossless","told":false,"design":"core","what":"The whole item typed on the language's standard keyboard layout while the computer is set to US English: every character is what the same key gives on a US keyboard. Every keystroke is kept, and for most text the key map can be undone: in the main run's build the round trip is exact for 196 of 200 Russian, 198 of 200 Hebrew, 178 of 200 Armenian and 186 of 200 Arabic items (draw ranks 50 to 249), and the contrasts use the exact items. Characters on keys that print nothing on the active layout are lost, and text that mixes two scripts can have more than one reading. The 15 Nov pilot used earlier letter tables (operator version 2): undoing them restores 280 of 300 Russian texts exactly. For English the typed form is a cipher, not another layout: every key shifted one position to the right on the US keyboard. Main run arm (a)."},{"id":"kb_layout_keys_ctrl","label":"Control sentence","short":"control sentence","kind":"lossless","told":false,"design":"core","what":"Wrong-layout text after a sentence that says only that the text was typed on a keyboard and is shown as typed. It names no layout. Main run arm (b')."},{"id":"kb_layout_keys_recog","label":"Another layout, not named","short":"other layout","kind":"lossless","told":true,"design":"core","what":"Wrong-layout text after a sentence that says a layout other than US English was intended, without naming it. Main run arm (b'')."},{"id":"kb_layout_keys_named","label":"Layout named","short":"layout named","kind":"lossless","told":true,"design":"core","what":"Wrong-layout text after a sentence that names the intended layout. Main run arm (b); in the pilot, one wording and no control sentence."},{"id":"kb_lk_phonetic","label":"Phonetic layout","short":"phonetic layout","kind":"lossless","told":false,"design":"core","what":"The language typed on its phonetic layout (letters on the Latin keys that sound alike) while the computer is on US English. Main run arm (c); Arabic has none."},{"id":"kb_lk_rand","label":"Random key map","short":"random key map","kind":"lossless","told":false,"design":"core","what":"The same text typed through a seeded random key map that carries the same information as the real layout: letters move among the letter keys (those on Latin-letter keys among those keys, those on punctuation keys among those); digits, punctuation and space stay put (on the English maps the digit keys move too). No sentence. Main run arm (d): one of ten seeds per item; the pilot used three seeds."},{"id":"kb_lk_rand_note","label":"Random map, rearrangement stated","short":"random map + note","kind":"lossless","told":true,"design":"core","what":"Random key map with a note that states the language and that the keys were rearranged; no table. Main run arm (e); in the pilot, the first map only."},{"id":"kb_lk_rand_table","label":"Key table given","short":"key table given","kind":"lossless","told":true,"design":"core","what":"Random key map with the note and the map's full letter table. Main run arm (f); in the pilot, the first map only."},{"id":"kb_lk_rand_shamtable","label":"Sham table","short":"sham table","kind":"lossless","told":true,"design":"core","what":"Random key map with the note and a table of the same form, number of entries and about the same length as the real table, which says nothing about the map (for each typed character, where its key sits on the US keyboard). Every sentence of it is true. Main run arm (f0), the comparison for C3."},{"id":"kb_lk_rand_xtable","label":"Another map's table","short":"wrong table","kind":"lossless","told":true,"design":"exploratory","what":"Random key map with the note and the table of another seed of the same layout. Main run arm (f x): exploratory, run on some settings only."},{"id":"native_ctrl","label":"As written, with the control sentence","short":"as written + control","kind":"native","told":false,"design":"secondary","what":"The native text after the control sentence (which is true of native text as well)."},{"id":"native_nopassage","label":"No passage","short":"no passage","kind":"native","told":false,"design":"secondary","what":"Question and options only, without the passage: what a model answers without reading the passage."},{"id":"kb_clean","label":"Romanised or without diacritics","short":"romanised / no accents","kind":"lossy","told":false,"design":"exploratory","what":"Lossy typed form: non-Latin scripts romanised (operator D1), Latin scripts with their diacritics removed (operator D2). Several words can share one typed form. Items whose typed form is identical to the text as written are left out of a cell."},{"id":"kb_pinyin_tone","label":"Pinyin with tone numbers","short":"pinyin, tone numbers","kind":"lossy","told":false,"design":"exploratory","what":"Chinese typed as pinyin with tone numbers (reduced loss)."},{"id":"kb_pinyin_word","label":"Pinyin without tones, spaced","short":"toneless pinyin","kind":"lossy","told":false,"design":"exploratory","what":"Chinese typed as pinyin without tones, words separated by spaces."},{"id":"kb_pinyin_unspaced","label":"Pinyin without tones, unspaced","short":"toneless, unspaced","kind":"lossy","told":false,"design":"exploratory","what":"Toneless pinyin with no spaces between words."},{"id":"kb_pinyin_initials","label":"Pinyin initials","short":"initials","kind":"lossy","told":false,"design":"exploratory","what":"The first letter of each pinyin syllable."},{"id":"kb_telex","label":"Vietnamese Telex keystrokes","short":"Telex","kind":"lossless","told":false,"design":"exploratory","what":"Vietnamese as raw Telex keystrokes, before the input method composes the letters."},{"id":"kb_vni","label":"Vietnamese VNI keystrokes","short":"VNI","kind":"lossless","told":false,"design":"exploratory","what":"Vietnamese as raw VNI keystrokes (tone and vowel marks as digits)."},{"id":"kb_romaji_ime","label":"Japanese romaji","short":"romaji","kind":"lossy","told":false,"design":"exploratory","what":"Japanese as the romaji typed into an input method, before kana-kanji conversion."},{"id":"kb_layout_keys_spaced","label":"Wrong layout, spaced","short":"wrong layout, spaced","kind":"lossless","told":false,"design":"secondary","what":"Wrong-layout text with one space between the characters (three for a typed space)."},{"id":"native_spaced","label":"As written, spaced","short":"as written, spaced","kind":"native","told":false,"design":"secondary","what":"The native text with one space between the characters: the control for the spaced arm."},{"id":"kb_layout_keys_maybe","label":"Wrong layout, generic hint","short":"generic hint","kind":"lossless","told":true,"design":"secondary","what":"Wrong-layout text after a note that is true of any text: it may have been typed with another layout active, and if so should be read as meant."},{"id":"native_maybe","label":"As written, generic hint","short":"as written + hint","kind":"native","told":false,"design":"secondary","what":"The native text after the same generic hint: what the hint costs on native text."},{"id":"kb_layout_keys_demo3","label":"Wrong layout, three examples","short":"three examples","kind":"lossless","told":true,"design":"secondary","what":"Wrong-layout text after three pairs of typed and meant sentences of the same layout pair; no other note."},{"id":"kb_lk_rand_demo3","label":"Random map, three examples","short":"random map + examples","kind":"lossless","told":true,"design":"secondary","what":"Random-map text after three pairs of typed and meant sentences of the same map; no other note."},{"id":"native_optrev","label":"As written, options reversed","short":"options reversed","kind":"native","told":false,"design":"secondary","what":"The native item with its four options in reverse order."},{"id":"kb_layout_keys_optrev","label":"Wrong layout, options reversed","short":"wrong layout, reversed","kind":"lossless","told":false,"design":"secondary","what":"The wrong-layout item with its four options in reverse order."},{"id":"kb_layout_keys_rev_ru","label":"English typed on the Russian layout","short":"English on Russian","kind":"lossless","told":false,"design":"exploratory","what":"English text typed while the computer is set to the Russian layout: the reverse direction."},{"id":"kb_layout_keys_rev_il","label":"English typed on the Hebrew layout","short":"English on Hebrew","kind":"lossless","told":false,"design":"exploratory","what":"English text typed while the computer is set to the Hebrew layout: the reverse direction."},{"id":"kb_layout_keys_rev_ko","label":"English typed on the Korean layout","short":"English on Korean","kind":"lossless","told":false,"design":"exploratory","what":"English text typed while the computer is set to the Korean layout: the reverse direction."},{"id":"kb_layout_keys_rev_ara","label":"English typed on the Arabic layout","short":"English on Arabic","kind":"lossless","told":false,"design":"exploratory","what":"English text typed while the computer is set to the Arabic layout: the reverse direction."}],"kind_text":{"native":"as written","lossless":"every keystroke kept","lossy":"lossy: several words can share one typed form"},"settings":[{"id":"claude_fable_5=low","model_key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5=medium","model_key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5=high","model_key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5=xhigh","model_key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5=max","model_key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_1=low","model_key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_1=medium","model_key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_1=high","model_key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_1=xhigh","model_key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_1=max","model_key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_2=low","model_key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_2=medium","model_key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_2=high","model_key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_2=xhigh","model_key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_fable_5_2=max","model_key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_haiku_4_5=off","model_key":"claude_haiku_4_5","model_id":"claude-haiku-4-5-20251001","vendor":"Anthropic","family":"claude-haiku","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_haiku_5_5=off","model_key":"claude_haiku_5_5","model_id":"claude-haiku-5-5","vendor":"Anthropic","family":"claude-haiku","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["haiku55_off"],"pilot_output_budget":256,"has_cells":true,"in_map":true,"pilot_run_health":{"calls":1290,"no_reply":0,"refusal":117,"truncated":0,"unparseable":0}},{"id":"claude_haiku_5_5=low","model_key":"claude_haiku_5_5","model_id":"claude-haiku-5-5","vendor":"Anthropic","family":"claude-haiku","tier":0,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_haiku_5_5=medium","model_key":"claude_haiku_5_5","model_id":"claude-haiku-5-5","vendor":"Anthropic","family":"claude-haiku","tier":0,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_haiku_5_5=high","model_key":"claude_haiku_5_5","model_id":"claude-haiku-5-5","vendor":"Anthropic","family":"claude-haiku","tier":0,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["haiku55_high"],"pilot_output_budget":16000,"has_cells":true,"pilot_run_health":{"calls":1150,"no_reply":0,"refusal":18,"truncated":0,"unparseable":0}},{"id":"claude_opus_4_5=off","model_key":"claude_opus_4_5","model_id":"claude-opus-4-5-20251101","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_6=off","model_key":"claude_opus_4_6","model_id":"claude-opus-4-6","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_6=high","model_key":"claude_opus_4_6","model_id":"claude-opus-4-6","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_7=off","model_key":"claude_opus_4_7","model_id":"claude-opus-4-7","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_7=high","model_key":"claude_opus_4_7","model_id":"claude-opus-4-7","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_8=off","model_key":"claude_opus_4_8","model_id":"claude-opus-4-8","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_4_8=high","model_key":"claude_opus_4_8","model_id":"claude-opus-4-8","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5=low","model_key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5=medium","model_key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5=high","model_key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5=xhigh","model_key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5=max","model_key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5_5=low","model_key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":["opus55_low"],"pilot_output_budget":16000,"has_cells":true,"pilot_run_health":{"calls":1470,"no_reply":0,"refusal":20,"truncated":0,"unparseable":0}},{"id":"claude_opus_5_5=medium","model_key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5_5=high","model_key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5_5=xhigh","model_key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_opus_5_5=max","model_key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":16000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_4_5=off","model_key":"claude_sonnet_4_5","model_id":"claude-sonnet-4-5-20250929","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_4_6=off","model_key":"claude_sonnet_4_6","model_id":"claude-sonnet-4-6","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_4_6=high","model_key":"claude_sonnet_4_6","model_id":"claude-sonnet-4-6","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5=off","model_key":"claude_sonnet_5","model_id":"claude-sonnet-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5=low","model_key":"claude_sonnet_5","model_id":"claude-sonnet-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5=medium","model_key":"claude_sonnet_5","model_id":"claude-sonnet-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5=high","model_key":"claude_sonnet_5","model_id":"claude-sonnet-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5_5=off","model_key":"claude_sonnet_5_5","model_id":"claude-sonnet-5-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["sonnet55_off"],"pilot_output_budget":256,"has_cells":true,"pilot_run_health":{"calls":1470,"no_reply":0,"refusal":27,"truncated":0,"unparseable":0}},{"id":"claude_sonnet_5_5=low","model_key":"claude_sonnet_5_5","model_id":"claude-sonnet-5-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5_5=medium","model_key":"claude_sonnet_5_5","model_id":"claude-sonnet-5-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"claude_sonnet_5_5=high","model_key":"claude_sonnet_5_5","model_id":"claude-sonnet-5-5","vendor":"Anthropic","family":"claude-sonnet","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":16000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["sonnet55_high"],"pilot_output_budget":16000,"has_cells":true,"pilot_run_health":{"calls":1470,"no_reply":0,"refusal":27,"truncated":0,"unparseable":0}},{"id":"doubao_seed_2_0_lite=off","model_key":"doubao_seed_2_0_lite","model_id":"doubao-seed-2-0-lite-260428","vendor":"ByteDance","family":"doubao-seed-lite","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_0_lite=on","model_key":"doubao_seed_2_0_lite","model_id":"doubao-seed-2-0-lite-260428","vendor":"ByteDance","family":"doubao-seed-lite","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_lite=off","model_key":"doubao_seed_2_1_lite","model_id":"doubao-seed-2-1-lite-260915","vendor":"ByteDance","family":"doubao-seed-lite","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["doubaolite_off"],"pilot_output_budget":64,"has_cells":true,"in_map":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"doubao_seed_2_1_lite=on","model_key":"doubao_seed_2_1_lite","model_id":"doubao-seed-2-1-lite-260915","vendor":"ByteDance","family":"doubao-seed-lite","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["doubaolite_on"],"pilot_output_budget":32000,"has_cells":true,"pilot_run_health":{"calls":1500,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"doubao_seed_2_0_mini=off","model_key":"doubao_seed_2_0_mini","model_id":"doubao-seed-2-0-mini-260428","vendor":"ByteDance","family":"doubao-seed-mini","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_0_mini=on","model_key":"doubao_seed_2_0_mini","model_id":"doubao-seed-2-0-mini-260428","vendor":"ByteDance","family":"doubao-seed-mini","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_pro=off","model_key":"doubao_seed_2_1_pro","model_id":"doubao-seed-2-1-pro-260915","vendor":"ByteDance","family":"doubao-seed-pro","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["doubaopro_off"],"pilot_output_budget":64,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"doubao_seed_2_1_pro=on","model_key":"doubao_seed_2_1_pro","model_id":"doubao-seed-2-1-pro-260915","vendor":"ByteDance","family":"doubao-seed-pro","tier":1,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_pro_260628=off","model_key":"doubao_seed_2_1_pro_260628","model_id":"doubao-seed-2-1-pro-260628","vendor":"ByteDance","family":"doubao-seed-pro","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_pro_260628=on","model_key":"doubao_seed_2_1_pro_260628","model_id":"doubao-seed-2-1-pro-260628","vendor":"ByteDance","family":"doubao-seed-pro","tier":1,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_turbo=off","model_key":"doubao_seed_2_1_turbo","model_id":"doubao-seed-2-1-turbo-260628","vendor":"ByteDance","family":"doubao-seed-turbo","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"doubao_seed_2_1_turbo=on","model_key":"doubao_seed_2_1_turbo","model_id":"doubao-seed-2-1-turbo-260628","vendor":"ByteDance","family":"doubao-seed-turbo","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_1_flash=off","model_key":"deepseek_v4_1_flash","model_id":"deepseek-v4-1-flash-261017","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"open_weights":true,"open_weights_repo":"deepseek-ai/DeepSeek-V4.1-Flash-1017","level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["dsflash_off"],"pilot_output_budget":64,"has_cells":true,"in_map":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"deepseek_v4_1_flash=on","model_key":"deepseek_v4_1_flash","model_id":"deepseek-v4-1-flash-261017","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"open_weights":true,"open_weights_repo":"deepseek-ai/DeepSeek-V4.1-Flash-1017","level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["dsflash_on"],"pilot_output_budget":32000,"has_cells":true,"pilot_run_health":{"calls":1500,"no_reply":0,"refusal":0,"truncated":166,"unparseable":0}},{"id":"deepseek_v4_1_flash_260910=off","model_key":"deepseek_v4_1_flash_260910","model_id":"deepseek-v4-1-flash-260910","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_1_flash_260910=on","model_key":"deepseek_v4_1_flash_260910","model_id":"deepseek-v4-1-flash-260910","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_flash=off","model_key":"deepseek_v4_flash","model_id":"deepseek-v4-flash-ga-260731","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_flash=on","model_key":"deepseek_v4_flash","model_id":"deepseek-v4-flash-ga-260731","vendor":"DeepSeek","family":"deepseek-flash","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_1_flash_economy=off","model_key":"deepseek_v4_1_flash_economy","model_id":"deepseek-v4-1-flash-economy-261017","vendor":"DeepSeek","family":"deepseek-flash-economy","tier":0,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_1_flash_economy=on","model_key":"deepseek_v4_1_flash_economy","model_id":"deepseek-v4-1-flash-economy-261017","vendor":"DeepSeek","family":"deepseek-flash-economy","tier":0,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"deepseek_v4_pro=off","model_key":"deepseek_v4_pro","model_id":"deepseek-v4-pro-ga-260813","vendor":"DeepSeek","family":"deepseek-pro","tier":1,"open_weights":true,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["dspro_off"],"pilot_output_budget":64,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"deepseek_v4_pro=on","model_key":"deepseek_v4_pro","model_id":"deepseek-v4-pro-ga-260813","vendor":"DeepSeek","family":"deepseek-pro","tier":1,"open_weights":true,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"kimi_k2_6_moonshot=off","model_key":"kimi_k2_6_moonshot","model_id":"kimi-k2.6","vendor":"Moonshot AI","family":"kimi","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"fail","smoke_note":"reasoning tokens with reasoning off","planned":false,"pilot_names":[],"has_cells":false},{"id":"kimi_k2_6_moonshot=on","model_key":"kimi_k2_6_moonshot","model_id":"kimi-k2.6","vendor":"Moonshot AI","family":"kimi","tier":1,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"kimi_k3_moonshot=off","model_key":"kimi_k3_moonshot","model_id":"kimi-k3","vendor":"Moonshot AI","family":"kimi","tier":1,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"kimi_k3_moonshot=on","model_key":"kimi_k3_moonshot","model_id":"kimi-k3","vendor":"Moonshot AI","family":"kimi","tier":1,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"kimi_k3_1_pro=off","model_key":"kimi_k3_1_pro","model_id":"kimi-k3-1-pro-261024","vendor":"Moonshot AI","family":"kimi-pro","tier":1,"open_weights":true,"open_weights_repo":"moonshotai/Kimi-K3.1-Pro","level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["kimi_off"],"pilot_output_budget":64,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"kimi_k3_1_pro=on","model_key":"kimi_k3_1_pro","model_id":"kimi-k3-1-pro-261024","vendor":"Moonshot AI","family":"kimi-pro","tier":1,"open_weights":true,"open_weights_repo":"moonshotai/Kimi-K3.1-Pro","level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["kimi_on"],"pilot_output_budget":32000,"has_cells":true,"pilot_run_health":{"calls":1500,"no_reply":0,"refusal":0,"truncated":47,"unparseable":0}},{"id":"kimi_k3_1_pro_moonshot=off","model_key":"kimi_k3_1_pro_moonshot","model_id":"kimi-k3.1-pro","vendor":"Moonshot AI","family":"kimi-pro","tier":1,"open_weights":true,"open_weights_repo":"moonshotai/Kimi-K3.1-Pro","level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass*","planned":true,"pilot_names":[],"has_cells":false},{"id":"kimi_k3_1_pro_moonshot=on","model_key":"kimi_k3_1_pro_moonshot","model_id":"kimi-k3.1-pro","vendor":"Moonshot AI","family":"kimi-pro","tier":1,"open_weights":true,"open_weights_repo":"moonshotai/Kimi-K3.1-Pro","level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_4_1=off","model_key":"gpt_4_1","model_id":"gpt-4.1","vendor":"OpenAI","family":"gpt","tier":1,"level":"off","level_label":"no reasoning","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5=minimal","model_key":"gpt_5","model_id":"gpt-5","vendor":"OpenAI","family":"gpt","tier":1,"level":"minimal","level_label":"effort minimal","level_order":2,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5=low","model_key":"gpt_5","model_id":"gpt-5","vendor":"OpenAI","family":"gpt","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5=medium","model_key":"gpt_5","model_id":"gpt-5","vendor":"OpenAI","family":"gpt","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5=high","model_key":"gpt_5","model_id":"gpt-5","vendor":"OpenAI","family":"gpt","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_2=off","model_key":"gpt_5_2","model_id":"gpt-5.2","vendor":"OpenAI","family":"gpt","tier":1,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_2=low","model_key":"gpt_5_2","model_id":"gpt-5.2","vendor":"OpenAI","family":"gpt","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_2=medium","model_key":"gpt_5_2","model_id":"gpt-5.2","vendor":"OpenAI","family":"gpt","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_2=high","model_key":"gpt_5_2","model_id":"gpt-5.2","vendor":"OpenAI","family":"gpt","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_4=off","model_key":"gpt_5_4","model_id":"gpt-5.4","vendor":"OpenAI","family":"gpt","tier":1,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_4=low","model_key":"gpt_5_4","model_id":"gpt-5.4","vendor":"OpenAI","family":"gpt","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_4=medium","model_key":"gpt_5_4","model_id":"gpt-5.4","vendor":"OpenAI","family":"gpt","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_4=high","model_key":"gpt_5_4","model_id":"gpt-5.4","vendor":"OpenAI","family":"gpt","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_5=off","model_key":"gpt_5_5","model_id":"gpt-5.5","vendor":"OpenAI","family":"gpt","tier":2,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_5=low","model_key":"gpt_5_5","model_id":"gpt-5.5","vendor":"OpenAI","family":"gpt","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_5=medium","model_key":"gpt_5_5","model_id":"gpt-5.5","vendor":"OpenAI","family":"gpt","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_5=high","model_key":"gpt_5_5","model_id":"gpt-5.5","vendor":"OpenAI","family":"gpt","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_4o=off","model_key":"gpt_4o","model_id":"gpt-4o","vendor":"OpenAI","family":"gpt-4o","tier":1,"level":"off","level_label":"no reasoning","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_4o_mini=off","model_key":"gpt_4o_mini","model_id":"gpt-4o-mini","vendor":"OpenAI","family":"gpt-4o-mini","tier":0,"level":"off","level_label":"no reasoning","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_astra=low","model_key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_astra=medium","model_key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_astra=high","model_key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_astra=xhigh","model_key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_astra=max","model_key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_astra=low","model_key":"gpt_6_astra","model_id":"gpt-6-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":["astra_low"],"pilot_output_budget":16000,"has_cells":true,"pilot_run_health":{"calls":1800,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_astra=medium","model_key":"gpt_6_astra","model_id":"gpt-6-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_astra=high","model_key":"gpt_6_astra","model_id":"gpt-6-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_astra=xhigh","model_key":"gpt_6_astra","model_id":"gpt-6-astra","vendor":"OpenAI","family":"gpt-astra","tier":2,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"expensive","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_luna=off","model_key":"gpt_5_6_luna","model_id":"gpt-5.6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_luna=low","model_key":"gpt_5_6_luna","model_id":"gpt-5.6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_luna=medium","model_key":"gpt_5_6_luna","model_id":"gpt-5.6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_luna=high","model_key":"gpt_5_6_luna","model_id":"gpt-5.6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_luna=off","model_key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["luna_none"],"pilot_output_budget":256,"has_cells":true,"in_map":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_luna=low","model_key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_luna=medium","model_key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_luna=high","model_key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":["luna_high"],"pilot_output_budget":32000,"has_cells":true,"in_map":true,"pilot_run_health":{"calls":2350,"no_reply":0,"refusal":0,"truncated":1,"unparseable":0}},{"id":"gpt_6_luna=xhigh","model_key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","tier":0,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"cheap","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_4_1_mini=off","model_key":"gpt_4_1_mini","model_id":"gpt-4.1-mini","vendor":"OpenAI","family":"gpt-mini","tier":0,"level":"off","level_label":"no reasoning","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_sol=off","model_key":"gpt_5_6_sol","model_id":"gpt-5.6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_sol=low","model_key":"gpt_5_6_sol","model_id":"gpt-5.6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_sol=medium","model_key":"gpt_5_6_sol","model_id":"gpt-5.6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_sol=high","model_key":"gpt_5_6_sol","model_id":"gpt-5.6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_1_sol=low","model_key":"gpt_6_1_sol","model_id":"gpt-6.1-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["sol_low"],"pilot_output_budget":16000,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_1_sol=medium","model_key":"gpt_6_1_sol","model_id":"gpt-6.1-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_1_sol=high","model_key":"gpt_6_1_sol","model_id":"gpt-6.1-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["sol_high"],"pilot_output_budget":32000,"has_cells":true,"pilot_run_health":{"calls":1800,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_1_sol=xhigh","model_key":"gpt_6_1_sol","model_id":"gpt-6.1-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_sol=low","model_key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_sol=medium","model_key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_sol=high","model_key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_sol=xhigh","model_key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_2_sol=max","model_key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"max","level_label":"effort max","level_order":7,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_sol=off","model_key":"gpt_6_sol","model_id":"gpt-6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_sol=low","model_key":"gpt_6_sol","model_id":"gpt-6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_sol=medium","model_key":"gpt_6_sol","model_id":"gpt-6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_sol=high","model_key":"gpt_6_sol","model_id":"gpt-6-sol","vendor":"OpenAI","family":"gpt-sol","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_terra=off","model_key":"gpt_5_6_terra","model_id":"gpt-5.6-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_terra=low","model_key":"gpt_5_6_terra","model_id":"gpt-5.6-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_terra=medium","model_key":"gpt_5_6_terra","model_id":"gpt-5.6-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_5_6_terra=high","model_key":"gpt_5_6_terra","model_id":"gpt-5.6-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_1_terra=off","model_key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"off","level_label":"effort none","level_order":0,"reasoning":"off","output_budget":256,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["terra_none"],"pilot_output_budget":256,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_1_terra=low","model_key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_1_terra=medium","model_key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"gpt_6_1_terra=high","model_key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":["terra_high"],"pilot_output_budget":32000,"has_cells":true,"pilot_run_health":{"calls":2450,"no_reply":0,"refusal":0,"truncated":0,"unparseable":0}},{"id":"gpt_6_1_terra=xhigh","model_key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","tier":0,"level":"xhigh","level_label":"effort xhigh","level_order":6,"reasoning":"on","output_budget":32000,"price_class":"middle","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"o3=low","model_key":"o3","model_id":"o3","vendor":"OpenAI","family":"o3","tier":1,"level":"low","level_label":"effort low","level_order":3,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"o3=medium","model_key":"o3","model_id":"o3","vendor":"OpenAI","family":"o3","tier":1,"level":"medium","level_label":"effort medium","level_order":4,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"o3=high","model_key":"o3","model_id":"o3","vendor":"OpenAI","family":"o3","tier":1,"level":"high","level_label":"effort high","level_order":5,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"glm_5_2=off","model_key":"glm_5_2","model_id":"glm-5-2-260617","vendor":"Z.ai","family":"glm","tier":1,"open_weights":true,"level":"off","level_label":"thinking off","level_order":0,"reasoning":"off","output_budget":256,"price_class":"unknown","roster":true,"smoke":"fail","smoke_note":"replies did not parse strictly","planned":false,"pilot_names":[],"has_cells":false},{"id":"glm_5_2=on","model_key":"glm_5_2","model_id":"glm-5-2-260617","vendor":"Z.ai","family":"glm","tier":1,"open_weights":true,"level":"on","level_label":"thinking on","level_order":8,"reasoning":"on","output_budget":32000,"price_class":"unknown","roster":true,"smoke":"pass","planned":true,"pilot_names":[],"has_cells":false},{"id":"glm_5_3_flash=off-requested","model_key":"glm_5_3_flash","model_id":"glm-5-3-flash-260828","vendor":"Z.ai","family":"glm-flash","tier":0,"open_weights":true,"open_weights_repo":"zai-org/GLM-5.3-Flash","level":"off-requested","level_label":"thinking off requested","level_order":1,"reasoning":"off-requested","output_budget":8000,"price_class":"cheap","roster":false,"smoke":"pass","planned":false,"pilot_names":["glmflash_offreq"],"pilot_output_budget":8000,"has_cells":true,"pilot_run_health":{"calls":1500,"no_reply":0,"refusal":0,"truncated":43,"unparseable":12}}],"models":[{"key":"claude_fable_5","model_id":"claude-fable-5","vendor":"Anthropic","family":"claude-fable","open_weights":null,"settings":["claude_fable_5=low","claude_fable_5=medium","claude_fable_5=high","claude_fable_5=xhigh","claude_fable_5=max"]},{"key":"claude_fable_5_1","model_id":"claude-fable-5-1","vendor":"Anthropic","family":"claude-fable","open_weights":null,"settings":["claude_fable_5_1=low","claude_fable_5_1=medium","claude_fable_5_1=high","claude_fable_5_1=xhigh","claude_fable_5_1=max"]},{"key":"claude_fable_5_2","model_id":"claude-fable-5-2","vendor":"Anthropic","family":"claude-fable","open_weights":null,"settings":["claude_fable_5_2=low","claude_fable_5_2=medium","claude_fable_5_2=high","claude_fable_5_2=xhigh","claude_fable_5_2=max"]},{"key":"claude_haiku_4_5","model_id":"claude-haiku-4-5-20251001","vendor":"Anthropic","family":"claude-haiku","open_weights":null,"settings":["claude_haiku_4_5=off"]},{"key":"claude_haiku_5_5","model_id":"claude-haiku-5-5","vendor":"Anthropic","family":"claude-haiku","open_weights":null,"settings":["claude_haiku_5_5=off","claude_haiku_5_5=low","claude_haiku_5_5=medium","claude_haiku_5_5=high"]},{"key":"claude_opus_4_5","model_id":"claude-opus-4-5-20251101","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_4_5=off"]},{"key":"claude_opus_4_6","model_id":"claude-opus-4-6","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_4_6=off","claude_opus_4_6=high"]},{"key":"claude_opus_4_7","model_id":"claude-opus-4-7","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_4_7=off","claude_opus_4_7=high"]},{"key":"claude_opus_4_8","model_id":"claude-opus-4-8","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_4_8=off","claude_opus_4_8=high"]},{"key":"claude_opus_5","model_id":"claude-opus-5","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_5=low","claude_opus_5=medium","claude_opus_5=high","claude_opus_5=xhigh","claude_opus_5=max"]},{"key":"claude_opus_5_5","model_id":"claude-opus-5-5","vendor":"Anthropic","family":"claude-opus","open_weights":null,"settings":["claude_opus_5_5=low","claude_opus_5_5=medium","claude_opus_5_5=high","claude_opus_5_5=xhigh","claude_opus_5_5=max"]},{"key":"claude_sonnet_4_5","model_id":"claude-sonnet-4-5-20250929","vendor":"Anthropic","family":"claude-sonnet","open_weights":null,"settings":["claude_sonnet_4_5=off"]},{"key":"claude_sonnet_4_6","model_id":"claude-sonnet-4-6","vendor":"Anthropic","family":"claude-sonnet","open_weights":null,"settings":["claude_sonnet_4_6=off","claude_sonnet_4_6=high"]},{"key":"claude_sonnet_5","model_id":"claude-sonnet-5","vendor":"Anthropic","family":"claude-sonnet","open_weights":null,"settings":["claude_sonnet_5=off","claude_sonnet_5=low","claude_sonnet_5=medium","claude_sonnet_5=high"]},{"key":"claude_sonnet_5_5","model_id":"claude-sonnet-5-5","vendor":"Anthropic","family":"claude-sonnet","open_weights":null,"settings":["claude_sonnet_5_5=off","claude_sonnet_5_5=low","claude_sonnet_5_5=medium","claude_sonnet_5_5=high"]},{"key":"doubao_seed_2_0_lite","model_id":"doubao-seed-2-0-lite-260428","vendor":"ByteDance","family":"doubao-seed-lite","open_weights":null,"settings":["doubao_seed_2_0_lite=off","doubao_seed_2_0_lite=on"]},{"key":"doubao_seed_2_1_lite","model_id":"doubao-seed-2-1-lite-260915","vendor":"ByteDance","family":"doubao-seed-lite","open_weights":null,"settings":["doubao_seed_2_1_lite=off","doubao_seed_2_1_lite=on"]},{"key":"doubao_seed_2_0_mini","model_id":"doubao-seed-2-0-mini-260428","vendor":"ByteDance","family":"doubao-seed-mini","open_weights":null,"settings":["doubao_seed_2_0_mini=off","doubao_seed_2_0_mini=on"]},{"key":"doubao_seed_2_1_pro","model_id":"doubao-seed-2-1-pro-260915","vendor":"ByteDance","family":"doubao-seed-pro","open_weights":null,"settings":["doubao_seed_2_1_pro=off","doubao_seed_2_1_pro=on"]},{"key":"doubao_seed_2_1_pro_260628","model_id":"doubao-seed-2-1-pro-260628","vendor":"ByteDance","family":"doubao-seed-pro","open_weights":null,"settings":["doubao_seed_2_1_pro_260628=off","doubao_seed_2_1_pro_260628=on"]},{"key":"doubao_seed_2_1_turbo","model_id":"doubao-seed-2-1-turbo-260628","vendor":"ByteDance","family":"doubao-seed-turbo","open_weights":null,"settings":["doubao_seed_2_1_turbo=off","doubao_seed_2_1_turbo=on"]},{"key":"deepseek_v4_1_flash","model_id":"deepseek-v4-1-flash-261017","vendor":"DeepSeek","family":"deepseek-flash","open_weights":true,"settings":["deepseek_v4_1_flash=off","deepseek_v4_1_flash=on"]},{"key":"deepseek_v4_1_flash_260910","model_id":"deepseek-v4-1-flash-260910","vendor":"DeepSeek","family":"deepseek-flash","open_weights":null,"settings":["deepseek_v4_1_flash_260910=off","deepseek_v4_1_flash_260910=on"]},{"key":"deepseek_v4_flash","model_id":"deepseek-v4-flash-ga-260731","vendor":"DeepSeek","family":"deepseek-flash","open_weights":null,"settings":["deepseek_v4_flash=off","deepseek_v4_flash=on"]},{"key":"deepseek_v4_1_flash_economy","model_id":"deepseek-v4-1-flash-economy-261017","vendor":"DeepSeek","family":"deepseek-flash-economy","open_weights":null,"settings":["deepseek_v4_1_flash_economy=off","deepseek_v4_1_flash_economy=on"]},{"key":"deepseek_v4_pro","model_id":"deepseek-v4-pro-ga-260813","vendor":"DeepSeek","family":"deepseek-pro","open_weights":true,"settings":["deepseek_v4_pro=off","deepseek_v4_pro=on"]},{"key":"kimi_k2_6_moonshot","model_id":"kimi-k2.6","vendor":"Moonshot AI","family":"kimi","open_weights":null,"settings":["kimi_k2_6_moonshot=off","kimi_k2_6_moonshot=on"]},{"key":"kimi_k3_moonshot","model_id":"kimi-k3","vendor":"Moonshot AI","family":"kimi","open_weights":null,"settings":["kimi_k3_moonshot=off","kimi_k3_moonshot=on"]},{"key":"kimi_k3_1_pro","model_id":"kimi-k3-1-pro-261024","vendor":"Moonshot AI","family":"kimi-pro","open_weights":true,"settings":["kimi_k3_1_pro=off","kimi_k3_1_pro=on"]},{"key":"kimi_k3_1_pro_moonshot","model_id":"kimi-k3.1-pro","vendor":"Moonshot AI","family":"kimi-pro","open_weights":true,"settings":["kimi_k3_1_pro_moonshot=off","kimi_k3_1_pro_moonshot=on"]},{"key":"gpt_4_1","model_id":"gpt-4.1","vendor":"OpenAI","family":"gpt","open_weights":null,"settings":["gpt_4_1=off"]},{"key":"gpt_5","model_id":"gpt-5","vendor":"OpenAI","family":"gpt","open_weights":null,"settings":["gpt_5=minimal","gpt_5=low","gpt_5=medium","gpt_5=high"]},{"key":"gpt_5_2","model_id":"gpt-5.2","vendor":"OpenAI","family":"gpt","open_weights":null,"settings":["gpt_5_2=off","gpt_5_2=low","gpt_5_2=medium","gpt_5_2=high"]},{"key":"gpt_5_4","model_id":"gpt-5.4","vendor":"OpenAI","family":"gpt","open_weights":null,"settings":["gpt_5_4=off","gpt_5_4=low","gpt_5_4=medium","gpt_5_4=high"]},{"key":"gpt_5_5","model_id":"gpt-5.5","vendor":"OpenAI","family":"gpt","open_weights":null,"settings":["gpt_5_5=off","gpt_5_5=low","gpt_5_5=medium","gpt_5_5=high"]},{"key":"gpt_4o","model_id":"gpt-4o","vendor":"OpenAI","family":"gpt-4o","open_weights":null,"settings":["gpt_4o=off"]},{"key":"gpt_4o_mini","model_id":"gpt-4o-mini","vendor":"OpenAI","family":"gpt-4o-mini","open_weights":null,"settings":["gpt_4o_mini=off"]},{"key":"gpt_6_2_astra","model_id":"gpt-6.2-astra","vendor":"OpenAI","family":"gpt-astra","open_weights":null,"settings":["gpt_6_2_astra=low","gpt_6_2_astra=medium","gpt_6_2_astra=high","gpt_6_2_astra=xhigh","gpt_6_2_astra=max"]},{"key":"gpt_6_astra","model_id":"gpt-6-astra","vendor":"OpenAI","family":"gpt-astra","open_weights":null,"settings":["gpt_6_astra=low","gpt_6_astra=medium","gpt_6_astra=high","gpt_6_astra=xhigh"]},{"key":"gpt_5_6_luna","model_id":"gpt-5.6-luna","vendor":"OpenAI","family":"gpt-luna","open_weights":null,"settings":["gpt_5_6_luna=off","gpt_5_6_luna=low","gpt_5_6_luna=medium","gpt_5_6_luna=high"]},{"key":"gpt_6_luna","model_id":"gpt-6-luna","vendor":"OpenAI","family":"gpt-luna","open_weights":null,"settings":["gpt_6_luna=off","gpt_6_luna=low","gpt_6_luna=medium","gpt_6_luna=high","gpt_6_luna=xhigh"]},{"key":"gpt_4_1_mini","model_id":"gpt-4.1-mini","vendor":"OpenAI","family":"gpt-mini","open_weights":null,"settings":["gpt_4_1_mini=off"]},{"key":"gpt_5_6_sol","model_id":"gpt-5.6-sol","vendor":"OpenAI","family":"gpt-sol","open_weights":null,"settings":["gpt_5_6_sol=off","gpt_5_6_sol=low","gpt_5_6_sol=medium","gpt_5_6_sol=high"]},{"key":"gpt_6_1_sol","model_id":"gpt-6.1-sol","vendor":"OpenAI","family":"gpt-sol","open_weights":null,"settings":["gpt_6_1_sol=low","gpt_6_1_sol=medium","gpt_6_1_sol=high","gpt_6_1_sol=xhigh"]},{"key":"gpt_6_2_sol","model_id":"gpt-6.2-sol","vendor":"OpenAI","family":"gpt-sol","open_weights":null,"settings":["gpt_6_2_sol=low","gpt_6_2_sol=medium","gpt_6_2_sol=high","gpt_6_2_sol=xhigh","gpt_6_2_sol=max"]},{"key":"gpt_6_sol","model_id":"gpt-6-sol","vendor":"OpenAI","family":"gpt-sol","open_weights":null,"settings":["gpt_6_sol=off","gpt_6_sol=low","gpt_6_sol=medium","gpt_6_sol=high"]},{"key":"gpt_5_6_terra","model_id":"gpt-5.6-terra","vendor":"OpenAI","family":"gpt-terra","open_weights":null,"settings":["gpt_5_6_terra=off","gpt_5_6_terra=low","gpt_5_6_terra=medium","gpt_5_6_terra=high"]},{"key":"gpt_6_1_terra","model_id":"gpt-6.1-terra","vendor":"OpenAI","family":"gpt-terra","open_weights":null,"settings":["gpt_6_1_terra=off","gpt_6_1_terra=low","gpt_6_1_terra=medium","gpt_6_1_terra=high","gpt_6_1_terra=xhigh"]},{"key":"o3","model_id":"o3","vendor":"OpenAI","family":"o3","open_weights":null,"settings":["o3=low","o3=medium","o3=high"]},{"key":"glm_5_2","model_id":"glm-5-2-260617","vendor":"Z.ai","family":"glm","open_weights":true,"settings":["glm_5_2=off","glm_5_2=on"]},{"key":"glm_5_3_flash","model_id":"glm-5-3-flash-260828","vendor":"Z.ai","family":"glm-flash","open_weights":true,"settings":["glm_5_3_flash=off-requested"]}],"languages":[{"code":"acm_Arab","bcp47":"acm-Arab","name_en":"Mesopotamian Arabic","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"afr_Latn","bcp47":"af-Latn","name_en":"Afrikaans","name_native":"Afrikaans","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"als_Latn","bcp47":"sq-Latn","name_en":"Albanian","name_native":"shqip","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"amh_Ethi","bcp47":"am-Ethi","name_en":"Amharic","name_native":"አማርኛ","script":"Ethi","script_name":"Ethiopic","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:amh"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"apc_Arab","bcp47":"apc-Arab","name_en":"North Levantine Arabic","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"arb_Arab","bcp47":"ar-Arab","name_en":"Arabic","name_native":"العربية","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth","ladder","pilot_layout"],"kind":"variant"},{"code":"arb_Latn","bcp47":"ar-Latn","name_en":"Arabic","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"ars_Arab","bcp47":"ars-Arab","name_en":"Arabic, Najdi","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"ary_Arab","bcp47":"ary-Arab","name_en":"Moroccan Arabic","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"arz_Arab","bcp47":"arz-Arab","name_en":"Egyptian Arabic","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ara>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"asm_Beng","bcp47":"as-Beng","name_en":"Assamese","name_native":"অসমীয়া","script":"Beng","script_name":"Bangla","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:asm"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"azj_Latn","bcp47":"az-Latn","name_en":"Azerbaijani","name_native":"azərbaycan","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"bam_Latn","bcp47":"bm-Latn","name_en":"Bambara","name_native":"bamanakan","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ben_Beng","bcp47":"bn-Beng","name_en":"Bangla","name_native":"বাংলা","script":"Beng","script_name":"Bangla","dir":"ltr","layout_pair":"in_ben>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:ben"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"ben_Latn","bcp47":"bn-Latn","name_en":"Bangla","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"bod_Tibt","bcp47":"bo-Tibt","name_en":"Tibetan","name_native":"བོད་སྐད་","script":"Tibt","script_name":"Tibetan","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:bod"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"bul_Cyrl","bcp47":"bg-Cyrl","name_en":"Bulgarian","name_native":"български","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"bg>us","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"cat_Latn","bcp47":"ca-Latn","name_en":"Catalan","name_native":"català","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ceb_Latn","bcp47":"ceb-Latn","name_en":"Cebuano","name_native":"Cebuano","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ces_Latn","bcp47":"cs-Latn","name_en":"Czech","name_native":"čeština","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ckb_Arab","bcp47":"ckb-Arab","name_en":"Kurdish, Sorani","name_native":"کوردیی ناوەندی","script":"Arab","script_name":"Arabic","dir":"rtl","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"dan_Latn","bcp47":"da-Latn","name_en":"Danish","name_native":"dansk","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"deu_Latn","bcp47":"de-Latn","name_en":"German","name_native":"Deutsch","script":"Latn","script_name":"Latin","dir":"ltr","layout_pair":"de>us","layout_family":"near-identity","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"ell_Grek","bcp47":"el-Grek","name_en":"Greek","name_native":"Ελληνικά","script":"Grek","script_name":"Greek","dir":"ltr","layout_pair":"gr>us","layout_family":"greek","kb_clean":{"operator":"D1","scheme":"greeklish"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"eng_Latn","bcp47":"en-Latn","name_en":"English","name_native":"English","script":"Latn","script_name":"Latin","dir":"ltr","layout_pair":"us_shr>us","layout_family":"shift","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"est_Latn","bcp47":"et-Latn","name_en":"Estonian","name_native":"eesti","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"eus_Latn","bcp47":"eu-Latn","name_en":"Basque","name_native":"euskara","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"fin_Latn","bcp47":"fi-Latn","name_en":"Finnish","name_native":"suomi","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"fra_Latn","bcp47":"fr-Latn","name_en":"French","name_native":"français","script":"Latn","script_name":"Latin","dir":"ltr","layout_pair":"fr>us","layout_family":"near-identity","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"fuv_Latn","bcp47":"fuv-Latn","name_en":"Nigerian Fulfulde","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"gaz_Latn","bcp47":"om-Latn","name_en":"Oromo","name_native":"Oromoo","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"grn_Latn","bcp47":"gn-Latn","name_en":"Guarani","name_native":"avañe’ẽ","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"guj_Gujr","bcp47":"gu-Gujr","name_en":"Gujarati","name_native":"ગુજરાતી","script":"Gujr","script_name":"Gujarati","dir":"ltr","layout_pair":"in_guj>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:guj"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"hat_Latn","bcp47":"ht-Latn","name_en":"Haitian Creole","name_native":"Kreyòl Ayisyen","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"hau_Latn","bcp47":"ha-Latn","name_en":"Hausa","name_native":"Hausa","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"heb_Hebr","bcp47":"he-Hebr","name_en":"Hebrew","name_native":"עברית","script":"Hebr","script_name":"Hebrew","dir":"rtl","layout_pair":"il>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:heb"},"null_control":false,"already_romanized":false,"groups":["map","breadth","ladder","pilot_layout"],"kind":"variant"},{"code":"hin_Deva","bcp47":"hi-Deva","name_en":"Hindi","name_native":"हिन्दी","script":"Deva","script_name":"Devanagari","dir":"ltr","layout_pair":"in_deva>us","kb_clean":{"operator":"D1","scheme":"hinglish"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"hin_Latn","bcp47":"hi-Latn","name_en":"Hindi","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"hrv_Latn","bcp47":"hr-Latn","name_en":"Croatian","name_native":"hrvatski","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"hun_Latn","bcp47":"hu-Latn","name_en":"Hungarian","name_native":"magyar","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"hye_Armn","bcp47":"hy-Armn","name_en":"Armenian","name_native":"հայերեն","script":"Armn","script_name":"Armenian","dir":"ltr","layout_pair":"am>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:hye"},"null_control":false,"already_romanized":false,"groups":["map","breadth","ladder"],"kind":"variant"},{"code":"ibo_Latn","bcp47":"ig-Latn","name_en":"Igbo","name_native":"Igbo","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ilo_Latn","bcp47":"ilo-Latn","name_en":"Iloko","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ind_Latn","bcp47":"id-Latn","name_en":"Indonesian","name_native":"Indonesia","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"isl_Latn","bcp47":"is-Latn","name_en":"Icelandic","name_native":"íslenska","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ita_Latn","bcp47":"it-Latn","name_en":"Italian","name_native":"italiano","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"jav_Latn","bcp47":"jv-Latn","name_en":"Javanese","name_native":"Jawa","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"jpn_Jpan","bcp47":"ja-Jpan","name_en":"Japanese","name_native":"日本語","script":"Jpan","script_name":"Japanese","dir":"ltr","kb_clean":{"operator":"D1","scheme":"romaji_wapuro_hepburn:fugashi_unidic_lite"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kac_Latn","bcp47":"kac-Latn","name_en":"Kachin","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kan_Knda","bcp47":"kn-Knda","name_en":"Kannada","name_native":"ಕನ್ನಡ","script":"Knda","script_name":"Kannada","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:kan"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kat_Geor","bcp47":"ka-Geor","name_en":"Georgian","name_native":"ქართული","script":"Geor","script_name":"Georgian","dir":"ltr","layout_pair":"ge>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:kat"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"kaz_Cyrl","bcp47":"kk-Cyrl","name_en":"Kazakh","name_native":"қазақ тілі","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"kz>us","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"kea_Latn","bcp47":"kea-Latn","name_en":"Kabuverdianu","name_native":"kabuverdianu","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"khk_Cyrl","bcp47":"mn-Cyrl","name_en":"Mongolian","name_native":"монгол","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"khm_Khmr","bcp47":"km-Khmr","name_en":"Khmer","name_native":"ខ្មែរ","script":"Khmr","script_name":"Khmer","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:khm"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kin_Latn","bcp47":"rw-Latn","name_en":"Kinyarwanda","name_native":"Ikinyarwanda","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kir_Cyrl","bcp47":"ky-Cyrl","name_en":"Kyrgyz","name_native":"кыргызча","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"kor_Hang","bcp47":"ko-Hang","name_en":"Korean","name_native":"한국어","script":"Hang","script_name":"Hangul","dir":"ltr","layout_pair":"ko>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"romanized_rr"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"lao_Laoo","bcp47":"lo-Laoo","name_en":"Lao","name_native":"ລາວ","script":"Laoo","script_name":"Lao","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:lao"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"lin_Latn","bcp47":"ln-Latn","name_en":"Lingala","name_native":"lingála","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"lit_Latn","bcp47":"lt-Latn","name_en":"Lithuanian","name_native":"lietuvių","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"lug_Latn","bcp47":"lg-Latn","name_en":"Ganda","name_native":"Luganda","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"luo_Latn","bcp47":"luo-Latn","name_en":"Luo","name_native":"Dholuo","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"lvs_Latn","bcp47":"lv-Latn","name_en":"Latvian","name_native":"latviešu","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"mal_Mlym","bcp47":"ml-Mlym","name_en":"Malayalam","name_native":"മലയാളം","script":"Mlym","script_name":"Malayalam","dir":"ltr","layout_pair":"in_mal>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:mal"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"mar_Deva","bcp47":"mr-Deva","name_en":"Marathi","name_native":"मराठी","script":"Deva","script_name":"Devanagari","dir":"ltr","layout_pair":"in_deva>us","kb_clean":{"operator":"D1","scheme":"hinglish"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"mkd_Cyrl","bcp47":"mk-Cyrl","name_en":"Macedonian","name_native":"македонски","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"mk>us","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"mlt_Latn","bcp47":"mt-Latn","name_en":"Maltese","name_native":"Malti","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"mri_Latn","bcp47":"mri-Latn","name_en":"Māori","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"mya_Mymr","bcp47":"my-Mymr","name_en":"Burmese","name_native":"မြန်မာ","script":"Mymr","script_name":"Myanmar","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:mya"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"nld_Latn","bcp47":"nl-Latn","name_en":"Dutch","name_native":"Nederlands","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"nob_Latn","bcp47":"nb-Latn","name_en":"Norwegian Bokmål","name_native":"norsk bokmål","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"npi_Deva","bcp47":"npi-Deva","name_en":"Nepali","script":"Deva","script_name":"Devanagari","dir":"ltr","kb_clean":{"operator":"D1","scheme":"hinglish"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"npi_Latn","bcp47":"npi-Latn","name_en":"Nepali","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"nso_Latn","bcp47":"nso-Latn","name_en":"Northern Sotho","name_native":"Sesotho sa Leboa","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"nya_Latn","bcp47":"ny-Latn","name_en":"Nyanja","name_native":"Nyanja","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ory_Orya","bcp47":"ory-Orya","name_en":"Odia","script":"Orya","script_name":"Odia","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:ory"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"pan_Guru","bcp47":"pa-Guru","name_en":"Punjabi","name_native":"ਪੰਜਾਬੀ","script":"Guru","script_name":"Gurmukhi","dir":"ltr","layout_pair":"in_guru>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:pan"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"pbt_Arab","bcp47":"pbt-Arab","name_en":"Southern Pashto","script":"Arab","script_name":"Arabic","dir":"rtl","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"pes_Arab","bcp47":"fa-Arab","name_en":"Persian","name_native":"فارسی","script":"Arab","script_name":"Arabic","dir":"rtl","layout_pair":"ir>us","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"plt_Latn","bcp47":"mg-Latn","name_en":"Malagasy","name_native":"Malagasy","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"pol_Latn","bcp47":"pl-Latn","name_en":"Polish","name_native":"polski","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"por_Latn","bcp47":"pt-Latn","name_en":"Portuguese","name_native":"português","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ron_Latn","bcp47":"ro-Latn","name_en":"Romanian","name_native":"română","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"rus_Cyrl","bcp47":"ru-Cyrl","name_en":"Russian","name_native":"русский","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"ru>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth","ladder","pilot_layout"],"kind":"variant"},{"code":"shn_Mymr","bcp47":"shn-Mymr","name_en":"Shan","name_native":"တႆး","script":"Mymr","script_name":"Myanmar","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:shn"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"sin_Latn","bcp47":"si-Latn","name_en":"Sinhala","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"sin_Sinh","bcp47":"si-Sinh","name_en":"Sinhala","name_native":"සිංහල","script":"Sinh","script_name":"Sinhala","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:sin"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"slk_Latn","bcp47":"sk-Latn","name_en":"Slovak","name_native":"slovenčina","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"slv_Latn","bcp47":"sl-Latn","name_en":"Slovenian","name_native":"slovenščina","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"sna_Latn","bcp47":"sn-Latn","name_en":"Shona","name_native":"chiShona","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"snd_Arab","bcp47":"sd-Arab","name_en":"Sindhi","name_native":"سنڌي","script":"Arab","script_name":"Arabic","dir":"rtl","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"som_Latn","bcp47":"so-Latn","name_en":"Somali","name_native":"Soomaali","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"sot_Latn","bcp47":"st-Latn","name_en":"Southern Sotho","name_native":"Sesotho","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"spa_Latn","bcp47":"es-Latn","name_en":"Spanish","name_native":"español","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"srp_Cyrl","bcp47":"sr-Cyrl","name_en":"Serbian","name_native":"српски","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"rs>us","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"ssw_Latn","bcp47":"ss-Latn","name_en":"Swati","name_native":"siSwati","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"sun_Latn","bcp47":"su-Latn","name_en":"Sundanese","name_native":"Basa Sunda","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"swe_Latn","bcp47":"sv-Latn","name_en":"Swedish","name_native":"svenska","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"swh_Latn","bcp47":"sw-Latn","name_en":"Swahili","name_native":"Kiswahili","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tam_Taml","bcp47":"ta-Taml","name_en":"Tamil","name_native":"தமிழ்","script":"Taml","script_name":"Tamil","dir":"ltr","layout_pair":"in_tam>us","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:tam"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"tel_Telu","bcp47":"te-Telu","name_en":"Telugu","name_native":"తెలుగు","script":"Telu","script_name":"Telugu","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:tel"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tgk_Cyrl","bcp47":"tg-Cyrl","name_en":"Tajik","name_native":"тоҷикӣ","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tgl_Latn","bcp47":"fil-Latn","name_en":"Filipino","name_native":"Filipino","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tha_Thai","bcp47":"th-Thai","name_en":"Thai","name_native":"ไทย","script":"Thai","script_name":"Thai","dir":"ltr","layout_pair":"th>us","layout_family":"cross-script","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:tha"},"null_control":false,"already_romanized":false,"groups":["map","breadth","pilot_layout"],"kind":"variant"},{"code":"tir_Ethi","bcp47":"ti-Ethi","name_en":"Tigrinya","name_native":"ትግርኛ","script":"Ethi","script_name":"Ethiopic","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:tir"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tsn_Latn","bcp47":"tn-Latn","name_en":"Tswana","name_native":"Setswana","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tso_Latn","bcp47":"ts-Latn","name_en":"Tsonga","name_native":"Xitsonga","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"tur_Latn","bcp47":"tr-Latn","name_en":"Turkish","name_native":"Türkçe","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"ukr_Cyrl","bcp47":"uk-Cyrl","name_en":"Ukrainian","name_native":"українська","script":"Cyrl","script_name":"Cyrillic","dir":"ltr","layout_pair":"ua>us","kb_clean":{"operator":"D1","scheme":"translit_adhoc"},"null_control":false,"already_romanized":false,"groups":["map","breadth"],"kind":"variant"},{"code":"urd_Arab","bcp47":"ur-Arab","name_en":"Urdu","name_native":"اردو","script":"Arab","script_name":"Arabic","dir":"rtl","kb_clean":{"operator":"D1","scheme":"arabizi_skeleton"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"urd_Latn","bcp47":"ur-Latn","name_en":"Urdu","script":"Latn","script_name":"Latin","dir":"ltr","romanised_variant":true,"kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":true,"groups":["map"],"kind":"variant"},{"code":"uzn_Latn","bcp47":"uz-Latn","name_en":"Uzbek","name_native":"o‘zbek","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":true,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"vie_Latn","bcp47":"vi-Latn","name_en":"Vietnamese","name_native":"Tiếng Việt","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"war_Latn","bcp47":"war-Latn","name_en":"Waray","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"wol_Latn","bcp47":"wo-Latn","name_en":"Wolof","name_native":"Wolof","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"xho_Latn","bcp47":"xh-Latn","name_en":"Xhosa","name_native":"IsiXhosa","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"yor_Latn","bcp47":"yo-Latn","name_en":"Yoruba","name_native":"Èdè Yorùbá","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"zho_Hans","bcp47":"zh-Hans","name_en":"Chinese","name_native":"中文","script":"Hans","script_name":"Simplified Han","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:zho"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"zho_Hant","bcp47":"zh-Hant","name_en":"Chinese","name_native":"中文","script":"Hant","script_name":"Traditional Han","dir":"ltr","kb_clean":{"operator":"D1","scheme":"uroman_1.3.1.1:zho"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"zsm_Latn","bcp47":"ms-Latn","name_en":"Malay","name_native":"Bahasa Melayu","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"zul_Latn","bcp47":"zu-Latn","name_en":"Zulu","name_native":"isiZulu","script":"Latn","script_name":"Latin","dir":"ltr","kb_clean":{"operator":"D2","scheme":"nfd_strip+special_fold"},"null_control":false,"already_romanized":false,"groups":["map"],"kind":"variant"},{"code":"pool_cross6","name_en":"6 cross-script layouts, pooled","dir":"ltr","null_control":false,"already_romanized":false,"groups":["pilot_layout"],"kind":"pool","layout_family":"cross-script","members":["rus_Cyrl","kor_Hang","heb_Hebr","tha_Thai","arb_Arab","arz_Arab"],"note":"Russian, Korean, Hebrew, Thai, Arabic and Egyptian Arabic (the last two share one keyboard), 50 items each: the set behind Figure A."}],"cells":[{"setting":"gpt_6_astra=low","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":24,"acc":0.48,"ci":[0.347969,0.614885],"ci_method":"wilson","outcomes":{"correct":24,"wrong":26,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":29,"acc":0.58,"ci":[0.442332,0.706252],"ci_method":"wilson","outcomes":{"correct":29,"wrong":21,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"wrong":33,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":16,"acc":0.32,"ci":[0.20758,0.458106],"ci_method":"wilson","outcomes":{"correct":16,"wrong":34,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":24,"acc":0.48,"ci":[0.347969,0.614885],"ci_method":"wilson","outcomes":{"correct":24,"wrong":26,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"wrong":24,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":31,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":30,"acc":0.6,"ci":[0.461812,0.723918],"ci_method":"wilson","outcomes":{"correct":30,"wrong":20,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":33,"acc":0.66,"ci":[0.521536,0.775632],"ci_method":"wilson","outcomes":{"correct":33,"wrong":17,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"wrong":11,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":27,"acc":0.54,"ci":[0.403986,0.670306],"ci_method":"wilson","outcomes":{"correct":27,"wrong":23,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":35,"acc":0.7,"ci":[0.562494,0.808966],"ci_method":"wilson","outcomes":{"correct":35,"wrong":15,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_lite=on","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"wrong":6,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"wrong":6,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":29,"acc":0.58,"ci":[0.442332,0.706252],"ci_method":"wilson","outcomes":{"correct":29,"wrong":21,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":27,"acc":0.54,"ci":[0.403986,0.670306],"ci_method":"wilson","outcomes":{"correct":27,"wrong":23,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"wrong":33,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":15,"acc":0.3,"ci":[0.191034,0.437506],"ci_method":"wilson","outcomes":{"correct":15,"wrong":35,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":31,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":28,"acc":0.56,"ci":[0.423058,0.68838],"ci_method":"wilson","outcomes":{"correct":28,"wrong":22,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"doubao_seed_2_1_pro=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":36,"acc":0.72,"ci":[0.583346,0.82526],"ci_method":"wilson","outcomes":{"correct":36,"wrong":14,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":49,"acc":1.0,"ci":[0.9273,1.0],"ci_method":"wilson","outcomes":{"correct":49,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":38,"acc":0.76,"ci":[0.625871,0.857027],"ci_method":"wilson","outcomes":{"correct":38,"wrong":12,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":24,"acc":0.48,"ci":[0.347969,0.614885],"ci_method":"wilson","outcomes":{"correct":24,"wrong":26,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":31,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":12,"acc":0.24,"ci":[0.142973,0.374129],"ci_method":"wilson","outcomes":{"correct":12,"wrong":38,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_1_flash=on","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"refused":0},"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"refused":0},"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":14,"acc":0.28,"ci":[0.17474,0.416654],"ci_method":"wilson","outcomes":{"correct":14,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":11,"acc":0.22,"ci":[0.127538,0.352418],"ci_method":"wilson","outcomes":{"correct":11,"refused":0},"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":1,"acc":0.02,"ci":[0.003539,0.104957],"ci_method":"wilson","outcomes":{"correct":1,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"refused":0},"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":9,"acc":0.18,"ci":[0.097701,0.307963],"ci_method":"wilson","outcomes":{"correct":9,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 164 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_1_flash=on","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"deepseek_v4_pro=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"wrong":10,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"wrong":10,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":10,"acc":0.2,"ci":[0.112436,0.330374],"ci_method":"wilson","outcomes":{"correct":10,"wrong":40,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":31,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":11,"acc":0.22,"ci":[0.127538,0.352418],"ci_method":"wilson","outcomes":{"correct":11,"wrong":39,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"deepseek_v4_pro=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"wrong":6,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"glm_5_3_flash=off-requested","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":38,"acc":0.76,"ci":[0.625871,0.857027],"ci_method":"wilson","outcomes":{"correct":38,"refused":0},"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":46,"acc":0.938776,"ci":[0.834793,0.97896],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"refused":0},"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"refused":0},"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"refused":0},"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":12,"acc":0.24,"ci":[0.142973,0.374129],"ci_method":"wilson","outcomes":{"correct":12,"refused":0},"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":11,"acc":0.22,"ci":[0.127538,0.352418],"ci_method":"wilson","outcomes":{"correct":11,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"refused":0},"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"refused":0},"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":10,"acc":0.2,"ci":[0.112436,0.330374],"ci_method":"wilson","outcomes":{"correct":10,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 43 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":41,"acc":0.82,"ci":[0.692037,0.902299],"ci_method":"wilson","outcomes":{"correct":41,"refused":0},"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"refused":0},"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"glm_5_3_flash=off-requested","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"claude_haiku_5_5=high","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":37,"acc":0.74,"ci":[0.604466,0.841286],"ci_method":"wilson","outcomes":{"correct":37,"wrong":13,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":16,"acc":0.32,"ci":[0.20758,0.458106],"ci_method":"wilson","outcomes":{"correct":16,"wrong":16,"refused":18,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":18,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":27,"acc":0.54,"ci":[0.403986,0.670306],"ci_method":"wilson","outcomes":{"correct":27,"wrong":23,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=high","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":24,"acc":0.48,"ci":[0.347969,0.614885],"ci_method":"wilson","outcomes":{"correct":24,"wrong":26,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"wrong":11,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":26,"refused":4,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":4,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":32,"acc":0.64,"ci":[0.501408,0.758614],"ci_method":"wilson","outcomes":{"correct":32,"wrong":18,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":5,"acc":0.25,"ci":[0.11186,0.468705],"ci_method":"wilson","outcomes":{"correct":5,"wrong":1,"refused":14,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":14,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":3,"acc":0.15,"ci":[0.052368,0.360423],"ci_method":"wilson","outcomes":{"correct":3,"wrong":3,"refused":14,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":14,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":34,"acc":0.68,"ci":[0.541894,0.79242],"ci_method":"wilson","outcomes":{"correct":34,"wrong":16,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":0,"refused":20,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":20,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":2,"acc":0.1,"ci":[0.027866,0.301038],"ci_method":"wilson","outcomes":{"correct":2,"wrong":4,"refused":14,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":14,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":1,"refused":19,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":19,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":3,"refused":17,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":17,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":1,"acc":0.05,"ci":[0.008881,0.236136],"ci_method":"wilson","outcomes":{"correct":1,"wrong":4,"refused":15,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":15,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_haiku_5_5=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":27,"acc":0.54,"ci":[0.403986,0.670306],"ci_method":"wilson","outcomes":{"correct":27,"wrong":23,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"wrong":33,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":30,"acc":0.6,"ci":[0.461812,0.723918],"ci_method":"wilson","outcomes":{"correct":30,"wrong":20,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":31,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":31,"acc":0.62,"ci":[0.481502,0.741374],"ci_method":"wilson","outcomes":{"correct":31,"wrong":19,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":16,"acc":0.32,"ci":[0.20758,0.458106],"ci_method":"wilson","outcomes":{"correct":16,"wrong":34,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"kimi_k3_1_pro=on","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":7,"acc":0.14,"ci":[0.069507,0.261865],"ci_method":"wilson","outcomes":{"correct":7,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":41,"acc":0.82,"ci":[0.692037,0.902299],"ci_method":"wilson","outcomes":{"correct":41,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","note":"Over the six cross-script layouts, 45 of 300 wrong-layout calls of this setting were cut off at the output budget; the summary does not split them by language.","outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"kimi_k3_1_pro=on","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"refused":0},"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":38,"acc":0.76,"ci":[0.625871,0.857027],"ci_method":"wilson","outcomes":{"correct":38,"refused":0},"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":32,"acc":0.64,"ci":[0.501408,0.758614],"ci_method":"wilson","outcomes":{"correct":32,"refused":0},"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":30,"acc":0.6,"ci":[0.461812,0.723918],"ci_method":"wilson","outcomes":{"correct":30,"refused":0},"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":35,"acc":0.7,"ci":[0.562494,0.808966],"ci_method":"wilson","outcomes":{"correct":35,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":34,"acc":0.68,"ci":[0.541894,0.79242],"ci_method":"wilson","outcomes":{"correct":34,"refused":0},"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"refused":0},"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":22,"acc":0.44,"ci":[0.31162,0.576942],"ci_method":"wilson","outcomes":{"correct":22,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"refused":0},"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"refused":0},"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"refused":0},"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=high","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"refused":0},"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"not_split"},{"setting":"gpt_6_luna=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"wrong":24,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"wrong":6,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":14,"acc":0.28,"ci":[0.17474,0.416654],"ci_method":"wilson","outcomes":{"correct":14,"wrong":36,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"wrong":24,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","outcomes":{"correct":23,"wrong":27,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"wrong":33,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"wrong":33,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_luna=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"wrong":10,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":0,"refused":20,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":20,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":35,"acc":0.7,"ci":[0.562494,0.808966],"ci_method":"wilson","outcomes":{"correct":35,"wrong":15,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_opus_5_5=low","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"wrong":10,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=high","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"wrong":6,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_sol=low","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":47,"acc":0.959184,"ci":[0.862867,0.988735],"ci_method":"wilson","outcomes":{"correct":47,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":38,"acc":0.76,"ci":[0.625871,0.857027],"ci_method":"wilson","outcomes":{"correct":38,"wrong":12,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"wrong":8,"refused":3,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":3,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":29,"acc":0.58,"ci":[0.442332,0.706252],"ci_method":"wilson","outcomes":{"correct":29,"wrong":17,"refused":4,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":4,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":0,"refused":20,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":20,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":35,"acc":0.7,"ci":[0.562494,0.808966],"ci_method":"wilson","outcomes":{"correct":35,"wrong":15,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":40,"acc":0.8,"ci":[0.669626,0.887564],"ci_method":"wilson","outcomes":{"correct":40,"wrong":10,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=high","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":41,"acc":0.82,"ci":[0.692037,0.902299],"ci_method":"wilson","outcomes":{"correct":41,"wrong":9,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":41,"acc":0.82,"ci":[0.692037,0.902299],"ci_method":"wilson","outcomes":{"correct":41,"wrong":9,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":24,"refused":1,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":1,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":31,"acc":0.62,"ci":[0.481502,0.741374],"ci_method":"wilson","outcomes":{"correct":31,"wrong":19,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"wrong":28,"refused":3,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":3,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":14,"acc":0.28,"ci":[0.17474,0.416654],"ci_method":"wilson","outcomes":{"correct":14,"wrong":33,"refused":3,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":3,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":20,"calls":20,"correct":0,"acc":0.0,"ci":[0.0,0.16113],"ci_method":"wilson","outcomes":{"correct":0,"wrong":0,"refused":20,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":20,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"wrong":30,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":37,"acc":0.74,"ci":[0.604466,0.841286],"ci_method":"wilson","outcomes":{"correct":37,"wrong":13,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"claude_sonnet_5_5=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":43,"acc":0.86,"ci":[0.738135,0.930493],"ci_method":"wilson","outcomes":{"correct":43,"wrong":7,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"wrong":8,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"wrong":24,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"wrong":5,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"wrong":24,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=high","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arb_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arz_Arab","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.32,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"deu_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":49,"calls":49,"correct":48,"acc":0.979592,"ci":[0.893062,0.996388],"ci_method":"wilson","outcomes":{"correct":48,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 49 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"ell_Grek","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.49,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":7,"calls":7,"correct":7,"acc":1.0,"ci":[0.645661,1.0],"ci_method":"wilson","outcomes":{"correct":7,"wrong":0,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"note":"Items whose romanised or stripped text is identical to the text as written are left out: 7 of the 50 items remain, so this cell and the text as written rest on different items.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"fra_Latn","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"heb_Hebr","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":41,"acc":0.82,"ci":[0.692037,0.902299],"ci_method":"wilson","outcomes":{"correct":41,"wrong":9,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2783,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"kor_Hang","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4367,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"tha_Thai","arm":"kb_clean","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":39,"acc":0.78,"ci":[0.647582,0.872462],"ci_method":"wilson","outcomes":{"correct":39,"wrong":11,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.355,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arb_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.2917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arz_Arab","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","outcomes":{"correct":25,"wrong":25,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.41,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"deu_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"ell_Grek","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.4133,"method":"overlap"},"operator":"pilot_v2","note":"Greek is kept apart: its layout is phonetically aligned with QWERTY, so the wrong-layout text is close to a transliteration.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"fra_Latn","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.37,"method":"overlap"},"operator":"pilot_v2","note":"German and French are typed on QWERTZ and AZERTY and read as QWERTY: few keys differ, so little changes.","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"heb_Hebr","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"wrong":29,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"kor_Hang","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":35,"acc":0.7,"ci":[0.562494,0.808966],"ci_method":"wilson","outcomes":{"correct":35,"wrong":15,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.365,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3917,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"tha_Thai","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"wrong":32,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arb_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3017,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"arz_Arab","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.36,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"deu_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.29,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"ell_Grek","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3667,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3333,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"fra_Latn","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"wrong":1,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3633,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"heb_Hebr","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3217,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"kor_Hang","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","outcomes":{"correct":48,"wrong":2,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.24,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","outcomes":{"correct":47,"wrong":3,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.39,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_1_terra=off","lang":"tha_Thai","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"wrong":4,"refused":0,"truncated":0,"unparseable":0,"no_reply":0},"no_valid":0,"floor":{"value":0.3083,"method":"overlap"},"outcomes_from":"run_health"},{"setting":"gpt_6_astra=low","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":290,"acc":0.966667,"ci":[0.939738,0.981795],"ci_method":"wilson","reasoning_tokens":{"mean":3.326667},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_astra=low","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":277,"acc":0.923333,"ci":[0.887589,0.948373],"ci_method":"wilson","outcomes":{"correct":277,"wrong":23,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":105.13,"median":48.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"doubao_seed_2_1_lite=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":280,"acc":0.933333,"ci":[0.899276,0.956433],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"doubao_seed_2_1_lite=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":120,"acc":0.4,"ci":[0.346164,0.456364],"ci_method":"wilson","outcomes":{"correct":120,"wrong":180,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"doubao_seed_2_1_lite=on","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":284,"acc":0.946667,"ci":[0.91513,0.966908],"ci_method":"wilson","reasoning_tokens":{"mean":2034.623333},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"doubao_seed_2_1_lite=on","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":146,"acc":0.486667,"ci":[0.430633,0.543037],"ci_method":"wilson","outcomes":{"correct":146,"wrong":154,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":9587.013333,"median":7822.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"doubao_seed_2_1_pro=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":281,"acc":0.936667,"ci":[0.903208,0.959083],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"doubao_seed_2_1_pro=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":120,"acc":0.4,"ci":[0.346164,0.456364],"ci_method":"wilson","outcomes":{"correct":120,"wrong":180,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"deepseek_v4_1_flash=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":279,"acc":0.93,"ci":[0.895363,0.953763],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"deepseek_v4_1_flash=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":113,"acc":0.376667,"ci":[0.323719,0.432733],"ci_method":"wilson","outcomes":{"correct":113,"wrong":187,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"deepseek_v4_1_flash=on","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":284,"acc":0.946667,"ci":[0.91513,0.966908],"ci_method":"wilson","reasoning_tokens":{"mean":488.55},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"deepseek_v4_1_flash=on","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":108,"acc":0.36,"ci":[0.307768,0.415772],"ci_method":"wilson","outcomes":{"correct":108,"wrong":28,"refused":0,"truncated":164,"unparseable":0},"no_valid":164,"reasoning_tokens":{"mean":23229.15,"median":32000.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"deepseek_v4_pro=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":282,"acc":0.94,"ci":[0.90716,0.961714],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"deepseek_v4_pro=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":101,"acc":0.336667,"ci":[0.285554,0.391909],"ci_method":"wilson","outcomes":{"correct":101,"wrong":199,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"glm_5_3_flash=off-requested","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":271,"acc":0.903333,"ci":[0.864617,0.93185],"ci_method":"wilson","reasoning_tokens":{"mean":9.213333},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"glm_5_3_flash=off-requested","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":96,"acc":0.32,"ci":[0.269774,0.374777],"ci_method":"wilson","outcomes":{"correct":96,"wrong":161,"refused":0,"truncated":43,"unparseable":0},"no_valid":43,"reasoning_tokens":{"mean":1581.56,"median":178.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"claude_haiku_5_5=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":120,"calls":120,"correct":111,"acc":0.925,"ci":[0.863589,0.960043],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50. Both forms here rest on those 120 items.","subset":"the first 20 items of each cross-script layout"},{"setting":"claude_haiku_5_5=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":120,"calls":120,"correct":11,"acc":0.091667,"ci":[0.051957,0.15671],"ci_method":"wilson","outcomes":{"correct":11,"wrong":16,"refused":93,"truncated":0,"unparseable":0},"no_valid":93,"reasoning_tokens":{"mean":0.0,"median":0.0},"operator":"pilot_v2","note":"Claude settings were sent the first 20 items of each cross-script layout only, because many wrong-layout requests were refused (pilot summary, section 6b); the text as written was run on all 50. Both forms here rest on those 120 items.","subset":"the first 20 items of each cross-script layout","outcomes_from":"summary"},{"setting":"kimi_k3_1_pro=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":283,"acc":0.943333,"ci":[0.911133,0.964323],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"kimi_k3_1_pro=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":126,"acc":0.42,"ci":[0.365505,0.476518],"ci_method":"wilson","outcomes":{"correct":126,"wrong":174,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"kimi_k3_1_pro=on","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":285,"acc":0.95,"ci":[0.919152,0.969469],"ci_method":"wilson","reasoning_tokens":{"mean":210.3},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"kimi_k3_1_pro=on","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":234,"acc":0.78,"ci":[0.729746,0.823173],"ci_method":"wilson","outcomes":{"correct":234,"wrong":21,"refused":0,"truncated":45,"unparseable":0},"no_valid":45,"reasoning_tokens":{"mean":10894.076667,"median":7134.5},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_luna=high","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":281,"acc":0.936667,"ci":[0.903208,0.959083],"ci_method":"wilson","reasoning_tokens":{"mean":81.326667},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_luna=high","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":196,"acc":0.653333,"ci":[0.597847,0.704942],"ci_method":"wilson","outcomes":{"correct":196,"wrong":104,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":5155.526667,"median":4692.5},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_luna=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":277,"acc":0.923333,"ci":[0.887589,0.948373],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_luna=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":110,"acc":0.366667,"ci":[0.314141,0.422564],"ci_method":"wilson","outcomes":{"correct":110,"wrong":190,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"claude_opus_5_5=low","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":291,"acc":0.97,"ci":[0.943977,0.984138],"ci_method":"wilson","reasoning_tokens":{"mean":20.026667},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"claude_opus_5_5=low","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":264,"acc":0.88,"ci":[0.838341,0.91205],"ci_method":"wilson","outcomes":{"correct":264,"wrong":36,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":419.653333,"median":380.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_1_sol=high","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":288,"acc":0.96,"ci":[0.931395,0.976973],"ci_method":"wilson","reasoning_tokens":{"mean":26.2},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_1_sol=high","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":276,"acc":0.92,"ci":[0.883726,0.945654],"ci_method":"wilson","outcomes":{"correct":276,"wrong":24,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":632.04,"median":263.5},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_1_sol=low","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":286,"acc":0.953333,"ci":[0.923202,0.972001],"ci_method":"wilson","reasoning_tokens":{"mean":3.883333},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_1_sol=low","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":272,"acc":0.906667,"ci":[0.868414,0.934636],"ci_method":"wilson","outcomes":{"correct":272,"wrong":28,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":162.733333,"median":75.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"claude_sonnet_5_5=high","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":288,"acc":0.96,"ci":[0.931395,0.976973],"ci_method":"wilson","reasoning_tokens":{"mean":18.846667},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"claude_sonnet_5_5=high","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":212,"acc":0.706667,"ci":[0.652793,0.755314],"ci_method":"wilson","outcomes":{"correct":212,"wrong":81,"refused":7,"truncated":0,"unparseable":0},"no_valid":7,"reasoning_tokens":{"mean":1173.926667,"median":996.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"claude_sonnet_5_5=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":285,"acc":0.95,"ci":[0.919152,0.969469],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"claude_sonnet_5_5=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":133,"acc":0.443333,"ci":[0.388186,0.499914],"ci_method":"wilson","outcomes":{"correct":133,"wrong":161,"refused":6,"truncated":0,"unparseable":0},"no_valid":6,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_1_terra=high","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":284,"acc":0.946667,"ci":[0.91513,0.966908],"ci_method":"wilson","reasoning_tokens":{"mean":14.486667},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_1_terra=high","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":228,"acc":0.76,"ci":[0.708578,0.804848],"ci_method":"wilson","outcomes":{"correct":228,"wrong":72,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":971.616667,"median":434.5},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_1_terra=off","lang":"pool_cross6","arm":"native","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":284,"acc":0.946667,"ci":[0.91513,0.966908],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"floor":{"value":0.3202833333333333,"method":"overlap"}},{"setting":"gpt_6_1_terra=off","lang":"pool_cross6","arm":"kb_layout_keys","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":171,"acc":0.57,"ci":[0.51344,0.62479],"ci_method":"wilson","outcomes":{"correct":171,"wrong":129,"refused":0,"truncated":0,"unparseable":0},"no_valid":0,"reasoning_tokens":{"mean":0.0,"median":0.0},"floor":{"value":0.3480666666666667,"method":"overlap"},"operator":"pilot_v2","outcomes_from":"summary"},{"setting":"gpt_6_luna=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":122,"acc":0.406667,"ci":[0.352601,0.463093],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_luna=high","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":237,"acc":0.79,"ci":[0.740388,0.832279],"ci_method":"wilson","reasoning_tokens":{"mean":5318.6866666666665},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","reasoning_tokens":{"mean":2004.26},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_terra=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":260,"acc":0.866667,"ci":[0.823527,0.900534],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":47,"acc":0.94,"ci":[0.837827,0.979385],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_terra=high","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":281,"acc":0.936667,"ci":[0.903208,0.959083],"ci_method":"wilson","reasoning_tokens":{"mean":221.81666666666666},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","reasoning_tokens":{"mean":91.74},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_sol=low","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":277,"acc":0.923333,"ci":[0.887589,0.948373],"ci_method":"wilson","reasoning_tokens":{"mean":88.39333333333333},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":48,"acc":0.96,"ci":[0.865397,0.988961],"ci_method":"wilson","reasoning_tokens":{"mean":36.9},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"deepseek_v4_1_flash=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":97,"acc":0.323333,"ci":[0.272925,0.378209],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"deepseek_v4_pro=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":116,"acc":0.386667,"ci":[0.333323,0.442877],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":16,"acc":0.32,"ci":[0.20758,0.458106],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"doubao_seed_2_1_lite=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":108,"acc":0.36,"ci":[0.307768,0.415772],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"doubao_seed_2_1_pro=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":111,"acc":0.37,"ci":[0.317331,0.425956],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"kimi_k3_1_pro=off","lang":"pool_cross6","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":300,"calls":300,"correct":124,"acc":0.413333,"ci":[0.359048,0.46981],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_layout_keys_named","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":31,"acc":0.62,"ci":[0.481502,0.741374],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"operator":"pilot_v2","note":"For English the typed form is every key shifted one position to the right on the US keyboard: a cipher, not another layout. One wording of the naming sentence; no control sentence was run in the pilot."},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":50,"acc":0.333333,"ci":[0.262886,0.412104],"ci_method":"wilson_maps","outcomes":{"correct":50,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.36,0.32,0.32]},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":61,"acc":0.406667,"ci":[0.33134,0.486655],"ci_method":"wilson_maps","outcomes":{"correct":61,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.34,0.48,0.4]},{"setting":"gpt_6_luna=high","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":53,"acc":0.353333,"ci":[0.281371,0.43262],"ci_method":"wilson_maps","outcomes":{"correct":53,"truncated":0},"reasoning_tokens":{"mean":11165.693333},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.36,0.38,0.32]},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":147,"acc":0.98,"ci":[0.942852,0.993175],"ci_method":"wilson_maps","outcomes":{"correct":147,"truncated":0},"reasoning_tokens":{"mean":1427.42},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.98,1.0,0.96]},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":61,"acc":0.406667,"ci":[0.33134,0.486655],"ci_method":"wilson_maps","outcomes":{"correct":61,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.36,0.38,0.48]},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":144,"acc":0.96,"ci":[0.915485,0.981541],"ci_method":"wilson_maps","outcomes":{"correct":144,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.96,0.98,0.94]},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":114,"acc":0.76,"ci":[0.685707,0.821308],"ci_method":"wilson_maps","outcomes":{"correct":114,"truncated":0},"reasoning_tokens":{"mean":1377.806667},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.74,0.78,0.76]},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":149,"acc":0.993333,"ci":[0.963206,0.998822],"ci_method":"wilson_maps","outcomes":{"correct":149,"truncated":0},"reasoning_tokens":{"mean":91.273333},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[1.0,0.98,1.0]},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":117,"acc":0.78,"ci":[0.707175,0.838841],"ci_method":"wilson_maps","outcomes":{"correct":117,"truncated":0},"reasoning_tokens":{"mean":521.506667},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.74,0.78,0.82]},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":146,"acc":0.973333,"ci":[0.933445,0.989582],"ci_method":"wilson_maps","outcomes":{"correct":146,"truncated":0},"reasoning_tokens":{"mean":33.52},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.96,0.98,0.98]},{"setting":"gpt_6_1_sol=high","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":131,"acc":0.873333,"ci":[0.810632,0.917389],"ci_method":"wilson_maps","outcomes":{"correct":131,"truncated":0},"reasoning_tokens":{"mean":1877.58},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.82,0.86,0.94]},{"setting":"gpt_6_1_sol=high","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":149,"acc":0.993333,"ci":[0.963206,0.998822],"ci_method":"wilson_maps","outcomes":{"correct":149,"truncated":0},"reasoning_tokens":{"mean":93.066667},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.98,1.0,1.0]},{"setting":"gpt_6_astra=low","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":122,"acc":0.813333,"ci":[0.743441,0.867577],"ci_method":"wilson_maps","outcomes":{"correct":122,"truncated":0},"reasoning_tokens":{"mean":396.433333},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.8,0.8,0.84]},{"setting":"gpt_6_astra=low","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":150,"acc":1.0,"ci":[0.975029,1.0],"ci_method":"wilson_maps","outcomes":{"correct":150,"truncated":0},"reasoning_tokens":{"mean":24.706667},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[1.0,1.0,1.0]},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":40,"acc":0.266667,"ci":[0.202371,0.342616],"ci_method":"wilson_maps","outcomes":{"correct":40,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.34,0.14,0.32]},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":51,"acc":0.34,"ci":[0.269032,0.418959],"ci_method":"wilson_maps","outcomes":{"correct":51,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.32,0.34,0.36]},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":45,"acc":0.3,"ci":[0.232407,0.377581],"ci_method":"wilson_maps","outcomes":{"correct":45,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.28,0.34,0.28]},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":61,"acc":0.406667,"ci":[0.33134,0.486655],"ci_method":"wilson_maps","outcomes":{"correct":61,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.34,0.44,0.44]},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":66,"acc":0.44,"ci":[0.363044,0.519953],"ci_method":"wilson_maps","outcomes":{"correct":66,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.52,0.42,0.38]},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":74,"acc":0.493333,"ci":[0.414495,0.572504],"ci_method":"wilson_maps","outcomes":{"correct":74,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.52,0.5,0.46]},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":69,"acc":0.46,"ci":[0.382234,0.539763],"ci_method":"wilson_maps","outcomes":{"correct":69,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.5,0.36,0.52]},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":80,"acc":0.533333,"ci":[0.453661,0.611341],"ci_method":"wilson_maps","outcomes":{"correct":80,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.56,0.54,0.5]},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":53,"acc":0.353333,"ci":[0.281371,0.43262],"ci_method":"wilson_maps","outcomes":{"correct":53,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Mean of three class-preserving random key maps (pilot seeds).","per_map":[0.3,0.38,0.38]},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":150,"correct":85,"acc":0.566667,"ci":[0.486678,0.643325],"ci_method":"wilson_maps","outcomes":{"correct":85,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"Random English letter substitutions: mean of three random key maps of the US layout (pilot seeds).","per_map":[0.5,0.66,0.54]},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":14,"acc":0.28,"ci":[0.17474,0.416654],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_luna=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","outcomes":{"correct":20,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":22,"acc":0.44,"ci":[0.31162,0.576942],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_luna=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_luna=high","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":42,"acc":0.84,"ci":[0.714855,0.916627],"ci_method":"wilson","outcomes":{"correct":42,"truncated":0},"reasoning_tokens":{"mean":5744.04},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_luna=high","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"truncated":0},"reasoning_tokens":{"mean":1375.96},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":23,"acc":0.46,"ci":[0.329694,0.596014],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":44,"acc":0.88,"ci":[0.761949,0.943825],"ci_method":"wilson","outcomes":{"correct":44,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","reasoning_tokens":{"mean":993.72},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":45,"acc":0.9,"ci":[0.786395,0.956525],"ci_method":"wilson","outcomes":{"correct":45,"truncated":0},"reasoning_tokens":{"mean":219.68},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","reasoning_tokens":{"mean":98.3},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_terra=high","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":50,"acc":1.0,"ci":[0.92865,1.0],"ci_method":"wilson","outcomes":{"correct":50,"truncated":0},"reasoning_tokens":{"mean":137.72},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","reasoning_tokens":{"mean":482.76},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":46,"acc":0.92,"ci":[0.811615,0.968451],"ci_method":"wilson","outcomes":{"correct":46,"truncated":0},"reasoning_tokens":{"mean":74.98},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","reasoning_tokens":{"mean":36.56},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"gpt_6_1_sol=low","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":49,"acc":0.98,"ci":[0.895043,0.996461],"ci_method":"wilson","outcomes":{"correct":49,"truncated":0},"reasoning_tokens":{"mean":33.82},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":9,"acc":0.18,"ci":[0.097701,0.307963],"ci_method":"wilson","outcomes":{"correct":9,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":12,"acc":0.24,"ci":[0.142973,0.374129],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":18,"acc":0.36,"ci":[0.241386,0.498592],"ci_method":"wilson","outcomes":{"correct":18,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":16,"acc":0.32,"ci":[0.20758,0.458106],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":13,"acc":0.26,"ci":[0.158714,0.395534],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"deepseek_v4_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":22,"acc":0.44,"ci":[0.31162,0.576942],"ci_method":"wilson","outcomes":{"correct":22,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":22,"acc":0.44,"ci":[0.31162,0.576942],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":21,"acc":0.42,"ci":[0.293748,0.557668],"ci_method":"wilson","outcomes":{"correct":21,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":20,"acc":0.4,"ci":[0.276082,0.538188],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":14,"acc":0.28,"ci":[0.17474,0.416654],"ci_method":"wilson","outcomes":{"correct":14,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":19,"acc":0.38,"ci":[0.258626,0.518498],"ci_method":"wilson","outcomes":{"correct":19,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":17,"acc":0.34,"ci":[0.224368,0.478464],"ci_method":"wilson","outcomes":{"correct":17,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_note","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":25,"acc":0.5,"ci":[0.366443,0.633557],"ci_method":"wilson","reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."},{"setting":"kimi_k3_1_pro=off","lang":"eng_Latn","arm":"kb_lk_rand_table","scope":"full","run":"pilot_layout_20261115","status":"pilot","date":"2026-11-15","items":50,"calls":50,"correct":26,"acc":0.52,"ci":[0.385115,0.652031],"ci_method":"wilson","outcomes":{"correct":26,"truncated":0},"reasoning_tokens":{"mean":0.0},"note":"The first class-preserving random map of the pilot, one wording of the note."}],"contrasts":[{"id":"layout_minus_native|gpt_6_astra=low|pool_cross6","setting":"gpt_6_astra=low","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.043333,"ci":[-0.076667,-0.016667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|doubao_seed_2_1_lite=off|pool_cross6","setting":"doubao_seed_2_1_lite=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.533333,"ci":[-0.613333,-0.453333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|doubao_seed_2_1_lite=on|pool_cross6","setting":"doubao_seed_2_1_lite=on","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.46,"ci":[-0.546667,-0.37],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|doubao_seed_2_1_pro=off|pool_cross6","setting":"doubao_seed_2_1_pro=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.536667,"ci":[-0.626667,-0.443333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|deepseek_v4_1_flash=off|pool_cross6","setting":"deepseek_v4_1_flash=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.553333,"ci":[-0.616667,-0.483333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|deepseek_v4_1_flash=on|pool_cross6","setting":"deepseek_v4_1_flash=on","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.586667,"ci":[-0.636667,-0.533333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|deepseek_v4_pro=off|pool_cross6","setting":"deepseek_v4_pro=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.603333,"ci":[-0.686667,-0.523333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|glm_5_3_flash=off-requested|pool_cross6","setting":"glm_5_3_flash=off-requested","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.583333,"ci":[-0.65,-0.516667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|claude_haiku_5_5=off|pool_cross6","setting":"claude_haiku_5_5=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.833333,"ci":[-0.9,-0.758333],"ci_method":"boot_item","unit":"share","n":120,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","subset":"the first 20 items of each cross-script layout"},{"id":"layout_minus_native|kimi_k3_1_pro=off|pool_cross6","setting":"kimi_k3_1_pro=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.523333,"ci":[-0.603333,-0.446667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|kimi_k3_1_pro=on|pool_cross6","setting":"kimi_k3_1_pro=on","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.17,"ci":[-0.203333,-0.14],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_luna=high|pool_cross6","setting":"gpt_6_luna=high","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.283333,"ci":[-0.343333,-0.226667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_luna=off|pool_cross6","setting":"gpt_6_luna=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.556667,"ci":[-0.63,-0.48],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|claude_opus_5_5=low|pool_cross6","setting":"claude_opus_5_5=low","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.09,"ci":[-0.136667,-0.05],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_1_sol=high|pool_cross6","setting":"gpt_6_1_sol=high","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.04,"ci":[-0.063333,-0.02],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_1_sol=low|pool_cross6","setting":"gpt_6_1_sol=low","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.046667,"ci":[-0.073333,-0.02],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|claude_sonnet_5_5=high|pool_cross6","setting":"claude_sonnet_5_5=high","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.253333,"ci":[-0.31,-0.196667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|claude_sonnet_5_5=off|pool_cross6","setting":"claude_sonnet_5_5=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.506667,"ci":[-0.583333,-0.43],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_1_terra=high|pool_cross6","setting":"gpt_6_1_terra=high","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.186667,"ci":[-0.25,-0.123333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"layout_minus_native|gpt_6_1_terra=off|pool_cross6","setting":"gpt_6_1_terra=off","lang":"pool_cross6","a":"kb_layout_keys","b":"native","label":"wrong layout minus as written","est":-0.376667,"ci":[-0.446667,-0.306667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"named_minus_plain|gpt_6_luna=off|pool_cross6","setting":"gpt_6_luna=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.406667,"b_value":0.366667,"est":0.04,"ci":[-0.013333,0.093333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_luna=off|eng_Latn","setting":"gpt_6_luna=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.42,"b_value":0.52,"est":-0.1,"ci":[-0.28,0.08],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_luna=high|pool_cross6","setting":"gpt_6_luna=high","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.79,"b_value":0.653333,"est":0.136667,"ci":[0.083333,0.19],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_luna=high|eng_Latn","setting":"gpt_6_luna=high","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.92,"b_value":0.9,"est":0.02,"ci":[-0.08,0.12],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_terra=off|pool_cross6","setting":"gpt_6_1_terra=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.866667,"b_value":0.57,"est":0.296667,"ci":[0.226667,0.37],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_terra=off|eng_Latn","setting":"gpt_6_1_terra=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.94,"b_value":0.98,"est":-0.04,"ci":[-0.12,0.04],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_terra=high|pool_cross6","setting":"gpt_6_1_terra=high","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.936667,"b_value":0.76,"est":0.176667,"ci":[0.116667,0.236667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_terra=high|eng_Latn","setting":"gpt_6_1_terra=high","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":1.0,"b_value":0.98,"est":0.02,"ci":[0.0,0.06],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_sol=low|pool_cross6","setting":"gpt_6_1_sol=low","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.923333,"b_value":0.906667,"est":0.016667,"ci":[-0.006667,0.04],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|gpt_6_1_sol=low|eng_Latn","setting":"gpt_6_1_sol=low","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.96,"b_value":0.98,"est":-0.02,"ci":[-0.06,0.0],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|deepseek_v4_1_flash=off|pool_cross6","setting":"deepseek_v4_1_flash=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.323333,"b_value":0.376667,"est":-0.053333,"ci":[-0.12,0.013333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|deepseek_v4_1_flash=off|eng_Latn","setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.36,"b_value":0.36,"est":0.0,"ci":[-0.18,0.18],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|deepseek_v4_pro=off|pool_cross6","setting":"deepseek_v4_pro=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.386667,"b_value":0.336667,"est":0.05,"ci":[-0.026667,0.12],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|deepseek_v4_pro=off|eng_Latn","setting":"deepseek_v4_pro=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.32,"b_value":0.4,"est":-0.08,"ci":[-0.26,0.1],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|doubao_seed_2_1_lite=off|pool_cross6","setting":"doubao_seed_2_1_lite=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.36,"b_value":0.4,"est":-0.04,"ci":[-0.116667,0.033333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|doubao_seed_2_1_lite=off|eng_Latn","setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.46,"b_value":0.48,"est":-0.02,"ci":[-0.18,0.12],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|doubao_seed_2_1_pro=off|pool_cross6","setting":"doubao_seed_2_1_pro=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.37,"b_value":0.4,"est":-0.03,"ci":[-0.1,0.036667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|doubao_seed_2_1_pro=off|eng_Latn","setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.38,"b_value":0.42,"est":-0.04,"ci":[-0.18,0.12],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|kimi_k3_1_pro=off|pool_cross6","setting":"kimi_k3_1_pro=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.413333,"b_value":0.42,"est":-0.006667,"ci":[-0.076667,0.056667],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain|kimi_k3_1_pro=off|eng_Latn","setting":"kimi_k3_1_pro=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence","a_value":0.62,"b_value":0.6,"est":0.02,"ci":[-0.08,0.12],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One wording, no control sentence."},{"id":"named_minus_plain_rerun|gpt_6_luna=off|pool_cross6","setting":"gpt_6_luna=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence, side-by-side rerun","a_value":0.396667,"b_value":0.403333,"est":-0.006667,"ci":[-0.08,0.063333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One more call on the plain and the named prompt of each item, sent next to each other."},{"id":"named_minus_plain_rerun|gpt_6_luna=off|eng_Latn","setting":"gpt_6_luna=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence, side-by-side rerun","a_value":0.56,"b_value":0.46,"est":0.1,"ci":[-0.04,0.24],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One more call on the plain and the named prompt of each item, sent next to each other."},{"id":"named_minus_plain_rerun|gpt_6_1_terra=off|pool_cross6","setting":"gpt_6_1_terra=off","lang":"pool_cross6","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence, side-by-side rerun","a_value":0.853333,"b_value":0.576667,"est":0.276667,"ci":[0.216667,0.333333],"ci_method":"boot_item","unit":"share","n":300,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One more call on the plain and the named prompt of each item, sent next to each other."},{"id":"named_minus_plain_rerun|gpt_6_1_terra=off|eng_Latn","setting":"gpt_6_1_terra=off","lang":"eng_Latn","a":"kb_layout_keys_named","b":"kb_layout_keys","label":"layout named minus no sentence, side-by-side rerun","a_value":0.96,"b_value":1.0,"est":-0.04,"ci":[-0.1,0.0],"ci_method":"boot_item","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory","note":"One more call on the plain and the named prompt of each item, sent next to each other."},{"id":"real_minus_random|gpt_6_luna=off|rus_Cyrl","setting":"gpt_6_luna=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.34,"b_value":0.333333,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.006667,"ci":[-0.12,0.14],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_luna=off|eng_Latn","setting":"gpt_6_luna=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.52,"b_value":0.406667,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.113333,"ci":[-0.04,0.26],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_luna=high|rus_Cyrl","setting":"gpt_6_luna=high","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.86,"b_value":0.353333,"a_reasoning":1579.14,"b_reasoning":11165.693333,"est":0.506667,"ci":[0.36,0.64],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_luna=high|eng_Latn","setting":"gpt_6_luna=high","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.9,"b_value":0.98,"a_reasoning":1421.82,"b_reasoning":1427.42,"est":-0.08,"ci":[-0.173333,0.0],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_terra=off|rus_Cyrl","setting":"gpt_6_1_terra=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.94,"b_value":0.406667,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.533333,"ci":[0.42,0.653333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_terra=off|eng_Latn","setting":"gpt_6_1_terra=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.98,"b_value":0.96,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.02,"ci":[-0.033333,0.073333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_terra=high|rus_Cyrl","setting":"gpt_6_1_terra=high","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.92,"b_value":0.76,"a_reasoning":57.04,"b_reasoning":1377.806667,"est":0.16,"ci":[0.06,0.26],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_terra=high|eng_Latn","setting":"gpt_6_1_terra=high","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.98,"b_value":0.993333,"a_reasoning":84.22,"b_reasoning":91.273333,"est":-0.013333,"ci":[-0.04,0.0],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_sol=low|rus_Cyrl","setting":"gpt_6_1_sol=low","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.92,"b_value":0.78,"a_reasoning":51.42,"b_reasoning":521.506667,"est":0.14,"ci":[0.04,0.24],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_sol=low|eng_Latn","setting":"gpt_6_1_sol=low","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.98,"b_value":0.973333,"a_reasoning":36.12,"b_reasoning":33.52,"est":0.006667,"ci":[0.0,0.02],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_sol=high|rus_Cyrl","setting":"gpt_6_1_sol=high","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.94,"b_value":0.873333,"a_reasoning":122.3,"b_reasoning":1877.58,"est":0.066667,"ci":[-0.006667,0.14],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_1_sol=high|eng_Latn","setting":"gpt_6_1_sol=high","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.98,"b_value":0.993333,"a_reasoning":104.16,"b_reasoning":93.066667,"est":-0.013333,"ci":[-0.04,0.0],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_astra=low|rus_Cyrl","setting":"gpt_6_astra=low","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.94,"b_value":0.813333,"a_reasoning":40.78,"b_reasoning":396.433333,"est":0.126667,"ci":[0.02,0.233333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|gpt_6_astra=low|eng_Latn","setting":"gpt_6_astra=low","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":1.0,"b_value":1.0,"a_reasoning":29.52,"b_reasoning":24.706667,"est":0.0,"ci":[0.0,0.0],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|deepseek_v4_1_flash=off|rus_Cyrl","setting":"deepseek_v4_1_flash=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.42,"b_value":0.266667,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.153333,"ci":[0.0,0.3],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|deepseek_v4_1_flash=off|eng_Latn","setting":"deepseek_v4_1_flash=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.36,"b_value":0.34,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.02,"ci":[-0.106667,0.16],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|deepseek_v4_pro=off|rus_Cyrl","setting":"deepseek_v4_pro=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.4,"b_value":0.3,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.1,"ci":[-0.033333,0.233333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|deepseek_v4_pro=off|eng_Latn","setting":"deepseek_v4_pro=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.4,"b_value":0.406667,"a_reasoning":0.0,"b_reasoning":0.0,"est":-0.006667,"ci":[-0.133333,0.126667],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|doubao_seed_2_1_lite=off|rus_Cyrl","setting":"doubao_seed_2_1_lite=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.38,"b_value":0.44,"a_reasoning":0.0,"b_reasoning":0.0,"est":-0.06,"ci":[-0.193333,0.08],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|doubao_seed_2_1_lite=off|eng_Latn","setting":"doubao_seed_2_1_lite=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.48,"b_value":0.493333,"a_reasoning":0.0,"b_reasoning":0.0,"est":-0.013333,"ci":[-0.126667,0.106667],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|doubao_seed_2_1_pro=off|rus_Cyrl","setting":"doubao_seed_2_1_pro=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.46,"b_value":0.46,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.0,"ci":[-0.14,0.14],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|doubao_seed_2_1_pro=off|eng_Latn","setting":"doubao_seed_2_1_pro=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.42,"b_value":0.533333,"a_reasoning":0.0,"b_reasoning":0.0,"est":-0.113333,"ci":[-0.24,0.026667],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|kimi_k3_1_pro=off|rus_Cyrl","setting":"kimi_k3_1_pro=off","lang":"rus_Cyrl","a":"kb_layout_keys","b":"kb_lk_rand","label":"real layout minus random key maps","a_value":0.62,"b_value":0.353333,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.266667,"ci":[0.113333,0.413333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"},{"id":"real_minus_random|kimi_k3_1_pro=off|eng_Latn","setting":"kimi_k3_1_pro=off","lang":"eng_Latn","a":"kb_layout_keys","b":"kb_lk_rand","label":"keys shifted right minus random key maps","a_value":0.6,"b_value":0.566667,"a_reasoning":0.0,"b_reasoning":0.0,"est":0.033333,"ci":[-0.106667,0.173333],"ci_method":"boot_item_maps","unit":"share","n":50,"run":"pilot_layout_20261115","status":"pilot","family":"exploratory"}],"costs":[{"setting":"gpt_6_astra=low","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":308,"output_tokens_mean":15,"reasoning_tokens_mean":3,"cost_per_call":{"value":0.00384,"currency":"USD"},"latency_s":2.3},{"setting":"gpt_6_astra=low","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":421,"output_tokens_mean":80,"reasoning_tokens_mean":67,"cost_per_call":{"value":0.00821,"currency":"USD"},"latency_s":3.9,"operator":"pilot_v2"},{"setting":"gpt_6_astra=low","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":300,"input_tokens_mean":448,"output_tokens_mean":224,"reasoning_tokens_mean":211,"cost_per_call":{"value":0.01569,"currency":"USD"},"latency_s":7.5},{"setting":"gpt_6_astra=low","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":346,"output_tokens_mean":21,"reasoning_tokens_mean":8,"cost_per_call":{"value":0.0045,"currency":"USD"},"latency_s":2.4},{"setting":"doubao_seed_2_1_lite=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":496,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00041,"currency":"CNY"},"latency_s":1.1},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":900,"input_tokens_mean":598,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00049,"currency":"CNY"},"latency_s":1.2,"operator":"pilot_v2"},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":654,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00054,"currency":"CNY"},"latency_s":1.0},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":620,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00051,"currency":"CNY"},"latency_s":1.1},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":662,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00055,"currency":"CNY"},"latency_s":1.1},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":950,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00078,"currency":"CNY"},"latency_s":1.0},{"setting":"doubao_seed_2_1_lite=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":514,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00043,"currency":"CNY"},"latency_s":1.1},{"setting":"doubao_seed_2_1_lite=on","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":497,"output_tokens_mean":1869,"reasoning_tokens_mean":1863,"cost_per_call":{"value":0.00544,"currency":"CNY"},"latency_s":25.0},{"setting":"doubao_seed_2_1_lite=on","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":592,"output_tokens_mean":8275,"reasoning_tokens_mean":8269,"cost_per_call":{"value":0.02282,"currency":"CNY"},"latency_s":97.1,"operator":"pilot_v2"},{"setting":"doubao_seed_2_1_lite=on","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":515,"output_tokens_mean":3396,"reasoning_tokens_mean":3390,"cost_per_call":{"value":0.00958,"currency":"CNY"},"latency_s":43.7},{"setting":"doubao_seed_2_1_pro=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":496,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00314,"currency":"CNY"},"latency_s":1.9},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":591,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00371,"currency":"CNY"},"latency_s":1.9,"operator":"pilot_v2"},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":654,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00408,"currency":"CNY"},"latency_s":1.8},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":620,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00388,"currency":"CNY"},"latency_s":1.8},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":662,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00413,"currency":"CNY"},"latency_s":2.1},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":950,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00586,"currency":"CNY"},"latency_s":1.9},{"setting":"doubao_seed_2_1_pro=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":514,"output_tokens_mean":5,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00325,"currency":"CNY"},"latency_s":1.9},{"setting":"deepseek_v4_1_flash=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":354,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.0003,"currency":"CNY"},"latency_s":1.2},{"setting":"deepseek_v4_1_flash=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":900,"input_tokens_mean":467,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00039,"currency":"CNY"},"latency_s":1.2,"operator":"pilot_v2"},{"setting":"deepseek_v4_1_flash=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":519,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00043,"currency":"CNY"},"latency_s":1.1},{"setting":"deepseek_v4_1_flash=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":488,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00041,"currency":"CNY"},"latency_s":1.2},{"setting":"deepseek_v4_1_flash=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":528,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00044,"currency":"CNY"},"latency_s":1.3},{"setting":"deepseek_v4_1_flash=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":817,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00067,"currency":"CNY"},"latency_s":1.1},{"setting":"deepseek_v4_1_flash=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":383,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00033,"currency":"CNY"},"latency_s":1.1},{"setting":"deepseek_v4_1_flash=on","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":378,"output_tokens_mean":420,"reasoning_tokens_mean":413,"cost_per_call":{"value":0.00165,"currency":"CNY"},"latency_s":5.5},{"setting":"deepseek_v4_1_flash=on","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":484,"output_tokens_mean":14535,"reasoning_tokens_mean":14530,"cost_per_call":{"value":0.0469,"currency":"CNY"},"latency_s":96.7,"operator":"pilot_v2"},{"setting":"deepseek_v4_1_flash=on","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":407,"output_tokens_mean":1815,"reasoning_tokens_mean":1808,"cost_per_call":{"value":0.00613,"currency":"CNY"},"latency_s":15.2},{"setting":"deepseek_v4_pro=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":436,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00408,"currency":"CNY"},"latency_s":1.7},{"setting":"deepseek_v4_pro=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":542,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00504,"currency":"CNY"},"latency_s":1.7,"operator":"pilot_v2"},{"setting":"deepseek_v4_pro=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":601,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00558,"currency":"CNY"},"latency_s":1.5},{"setting":"deepseek_v4_pro=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":570,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00529,"currency":"CNY"},"latency_s":1.5},{"setting":"deepseek_v4_pro=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":610,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00565,"currency":"CNY"},"latency_s":1.7},{"setting":"deepseek_v4_pro=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":899,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00826,"currency":"CNY"},"latency_s":1.4},{"setting":"deepseek_v4_pro=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":465,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00434,"currency":"CNY"},"latency_s":1.6},{"setting":"glm_5_3_flash=off-requested","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":355,"output_tokens_mean":15,"reasoning_tokens_mean":8,"cost_per_call":{"value":0.00032,"currency":"CNY"},"latency_s":1.3},{"setting":"glm_5_3_flash=off-requested","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":394,"output_tokens_mean":1124,"reasoning_tokens_mean":1089,"cost_per_call":{"value":0.00346,"currency":"CNY"},"latency_s":20.5,"operator":"pilot_v2"},{"setting":"glm_5_3_flash=off-requested","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":325,"output_tokens_mean":38,"reasoning_tokens_mean":30,"cost_per_call":{"value":0.00037,"currency":"CNY"},"latency_s":1.8},{"setting":"claude_haiku_5_5=high","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":677,"output_tokens_mean":121,"reasoning_tokens_mean":110,"cost_per_call":{"value":0.00013,"currency":"USD"},"latency_s":1.6},{"setting":"claude_haiku_5_5=high","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":150,"input_tokens_mean":761,"output_tokens_mean":411,"reasoning_tokens_mean":400,"cost_per_call":{"value":0.00028,"currency":"USD"},"latency_s":3.3,"operator":"pilot_v2"},{"setting":"claude_haiku_5_5=high","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":675,"output_tokens_mean":599,"reasoning_tokens_mean":589,"cost_per_call":{"value":0.00037,"currency":"USD"},"latency_s":4.0},{"setting":"claude_haiku_5_5=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":676,"output_tokens_mean":10,"reasoning_tokens_mean":0,"cost_per_call":{"value":7e-05,"currency":"USD"},"latency_s":1.2},{"setting":"claude_haiku_5_5=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":290,"input_tokens_mean":772,"output_tokens_mean":6,"reasoning_tokens_mean":0,"cost_per_call":{"value":8e-05,"currency":"USD"},"latency_s":1.4,"operator":"pilot_v2"},{"setting":"claude_haiku_5_5=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":674,"output_tokens_mean":10,"reasoning_tokens_mean":0,"cost_per_call":{"value":7e-05,"currency":"USD"},"latency_s":1.3},{"setting":"kimi_k3_1_pro=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":463,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00709,"currency":"CNY"},"latency_s":1.8},{"setting":"kimi_k3_1_pro=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":900,"input_tokens_mean":500,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00758,"currency":"CNY"},"latency_s":2.5,"operator":"pilot_v2"},{"setting":"kimi_k3_1_pro=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":550,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00823,"currency":"CNY"},"latency_s":1.7},{"setting":"kimi_k3_1_pro=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":520,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00784,"currency":"CNY"},"latency_s":2.5},{"setting":"kimi_k3_1_pro=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":560,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00836,"currency":"CNY"},"latency_s":1.8},{"setting":"kimi_k3_1_pro=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":849,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.01212,"currency":"CNY"},"latency_s":1.9},{"setting":"kimi_k3_1_pro=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":431,"output_tokens_mean":16,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00669,"currency":"CNY"},"latency_s":1.8},{"setting":"kimi_k3_1_pro=on","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":530,"output_tokens_mean":206,"reasoning_tokens_mean":186,"cost_per_call":{"value":0.0209,"currency":"CNY"},"latency_s":10.2},{"setting":"kimi_k3_1_pro=on","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":563,"output_tokens_mean":6762,"reasoning_tokens_mean":6743,"cost_per_call":{"value":0.4671,"currency":"CNY"},"latency_s":351.9,"operator":"pilot_v2"},{"setting":"kimi_k3_1_pro=on","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":498,"output_tokens_mean":717,"reasoning_tokens_mean":697,"cost_per_call":{"value":0.05525,"currency":"CNY"},"latency_s":31.3},{"setting":"gpt_6_luna=high","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":82,"reasoning_tokens_mean":69,"cost_per_call":{"value":7e-05,"currency":"USD"},"latency_s":1.9},{"setting":"gpt_6_luna=high","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":411,"output_tokens_mean":3285,"reasoning_tokens_mean":3271,"cost_per_call":{"value":0.00168,"currency":"USD"},"latency_s":28.6,"operator":"pilot_v2"},{"setting":"gpt_6_luna=high","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":470,"output_tokens_mean":4859,"reasoning_tokens_mean":4845,"cost_per_call":{"value":0.00248,"currency":"USD"},"latency_s":45.1},{"setting":"gpt_6_luna=high","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":441,"output_tokens_mean":6479,"reasoning_tokens_mean":6465,"cost_per_call":{"value":0.00328,"currency":"USD"},"latency_s":58.9},{"setting":"gpt_6_luna=high","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":768,"output_tokens_mean":3574,"reasoning_tokens_mean":3560,"cost_per_call":{"value":0.00186,"currency":"USD"},"latency_s":34.2},{"setting":"gpt_6_luna=high","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":331,"reasoning_tokens_mean":317,"cost_per_call":{"value":0.0002,"currency":"USD"},"latency_s":3.9},{"setting":"gpt_6_luna=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":4e-05,"currency":"USD"},"latency_s":1.4},{"setting":"gpt_6_luna=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":1250,"input_tokens_mean":417,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":5e-05,"currency":"USD"},"latency_s":1.6,"operator":"pilot_v2"},{"setting":"gpt_6_luna=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":700,"input_tokens_mean":470,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":5e-05,"currency":"USD"},"latency_s":2.0},{"setting":"gpt_6_luna=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":441,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":5e-05,"currency":"USD"},"latency_s":1.5},{"setting":"gpt_6_luna=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":482,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":5e-05,"currency":"USD"},"latency_s":2.4},{"setting":"gpt_6_luna=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":768,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":8e-05,"currency":"USD"},"latency_s":1.4},{"setting":"gpt_6_luna=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":4e-05,"currency":"USD"},"latency_s":1.4},{"setting":"claude_opus_5_5=low","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":677,"output_tokens_mean":29,"reasoning_tokens_mean":18,"cost_per_call":{"value":0.00328,"currency":"USD"},"latency_s":1.7},{"setting":"claude_opus_5_5=low","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":470,"input_tokens_mean":776,"output_tokens_mean":286,"reasoning_tokens_mean":275,"cost_per_call":{"value":0.00882,"currency":"USD"},"latency_s":4.9,"operator":"pilot_v2"},{"setting":"claude_opus_5_5=low","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":675,"output_tokens_mean":53,"reasoning_tokens_mean":42,"cost_per_call":{"value":0.00376,"currency":"USD"},"latency_s":2.1},{"setting":"gpt_6_1_sol=high","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":34,"reasoning_tokens_mean":21,"cost_per_call":{"value":0.00094,"currency":"USD"},"latency_s":2.6},{"setting":"gpt_6_1_sol=high","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":411,"output_tokens_mean":412,"reasoning_tokens_mean":398,"cost_per_call":{"value":0.00494,"currency":"USD"},"latency_s":9.8,"operator":"pilot_v2"},{"setting":"gpt_6_1_sol=high","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":300,"input_tokens_mean":438,"output_tokens_mean":999,"reasoning_tokens_mean":985,"cost_per_call":{"value":0.01087,"currency":"USD"},"latency_s":19.5},{"setting":"gpt_6_1_sol=high","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":64,"reasoning_tokens_mean":51,"cost_per_call":{"value":0.00131,"currency":"USD"},"latency_s":3.0},{"setting":"gpt_6_1_sol=low","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":15,"reasoning_tokens_mean":3,"cost_per_call":{"value":0.00075,"currency":"USD"},"latency_s":2.2},{"setting":"gpt_6_1_sol=low","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":411,"output_tokens_mean":116,"reasoning_tokens_mean":102,"cost_per_call":{"value":0.00198,"currency":"USD"},"latency_s":4.4,"operator":"pilot_v2"},{"setting":"gpt_6_1_sol=low","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":470,"output_tokens_mean":95,"reasoning_tokens_mean":81,"cost_per_call":{"value":0.00189,"currency":"USD"},"latency_s":3.8},{"setting":"gpt_6_1_sol=low","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":441,"output_tokens_mean":286,"reasoning_tokens_mean":272,"cost_per_call":{"value":0.00374,"currency":"USD"},"latency_s":8.2},{"setting":"gpt_6_1_sol=low","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":482,"output_tokens_mean":274,"reasoning_tokens_mean":260,"cost_per_call":{"value":0.0037,"currency":"USD"},"latency_s":7.9},{"setting":"gpt_6_1_sol=low","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":768,"output_tokens_mean":68,"reasoning_tokens_mean":54,"cost_per_call":{"value":0.00222,"currency":"USD"},"latency_s":3.3},{"setting":"gpt_6_1_sol=low","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":23,"reasoning_tokens_mean":11,"cost_per_call":{"value":0.0009,"currency":"USD"},"latency_s":2.4},{"setting":"claude_sonnet_5_5=high","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":677,"output_tokens_mean":26,"reasoning_tokens_mean":15,"cost_per_call":{"value":0.00161,"currency":"USD"},"latency_s":1.6},{"setting":"claude_sonnet_5_5=high","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":470,"input_tokens_mean":776,"output_tokens_mean":784,"reasoning_tokens_mean":773,"cost_per_call":{"value":0.00939,"currency":"USD"},"latency_s":8.4,"operator":"pilot_v2"},{"setting":"claude_sonnet_5_5=high","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":675,"output_tokens_mean":194,"reasoning_tokens_mean":183,"cost_per_call":{"value":0.00329,"currency":"USD"},"latency_s":3.0},{"setting":"claude_sonnet_5_5=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":676,"output_tokens_mean":10,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00145,"currency":"USD"},"latency_s":1.5},{"setting":"claude_sonnet_5_5=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":470,"input_tokens_mean":775,"output_tokens_mean":10,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00165,"currency":"USD"},"latency_s":1.7,"operator":"pilot_v2"},{"setting":"claude_sonnet_5_5=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":674,"output_tokens_mean":10,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00145,"currency":"USD"},"latency_s":1.6},{"setting":"gpt_6_1_terra=high","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":25,"reasoning_tokens_mean":12,"cost_per_call":{"value":0.00042,"currency":"USD"},"latency_s":1.7},{"setting":"gpt_6_1_terra=high","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":411,"output_tokens_mean":610,"reasoning_tokens_mean":597,"cost_per_call":{"value":0.00346,"currency":"USD"},"latency_s":9.8,"operator":"pilot_v2"},{"setting":"gpt_6_1_terra=high","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":350,"input_tokens_mean":470,"output_tokens_mean":217,"reasoning_tokens_mean":203,"cost_per_call":{"value":0.00156,"currency":"USD"},"latency_s":4.6},{"setting":"gpt_6_1_terra=high","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":441,"output_tokens_mean":748,"reasoning_tokens_mean":734,"cost_per_call":{"value":0.00418,"currency":"USD"},"latency_s":11.7},{"setting":"gpt_6_1_terra=high","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":482,"output_tokens_mean":560,"reasoning_tokens_mean":546,"cost_per_call":{"value":0.00328,"currency":"USD"},"latency_s":9.2},{"setting":"gpt_6_1_terra=high","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":768,"output_tokens_mean":193,"reasoning_tokens_mean":179,"cost_per_call":{"value":0.00173,"currency":"USD"},"latency_s":3.8},{"setting":"gpt_6_1_terra=high","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":58,"reasoning_tokens_mean":45,"cost_per_call":{"value":0.00062,"currency":"USD"},"latency_s":2.1},{"setting":"gpt_6_1_terra=off","arm":"native","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":298,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00036,"currency":"USD"},"latency_s":1.5},{"setting":"gpt_6_1_terra=off","arm":"kb_layout_keys","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":1250,"input_tokens_mean":417,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00048,"currency":"USD"},"latency_s":1.6,"operator":"pilot_v2"},{"setting":"gpt_6_1_terra=off","arm":"kb_layout_keys_named","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":700,"input_tokens_mean":470,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00053,"currency":"USD"},"latency_s":1.7},{"setting":"gpt_6_1_terra=off","arm":"kb_lk_rand","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":400,"input_tokens_mean":441,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.0005,"currency":"USD"},"latency_s":1.8},{"setting":"gpt_6_1_terra=off","arm":"kb_lk_rand_note","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":482,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00054,"currency":"USD"},"latency_s":1.7},{"setting":"gpt_6_1_terra=off","arm":"kb_lk_rand_table","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":100,"input_tokens_mean":768,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.00083,"currency":"USD"},"latency_s":1.6},{"setting":"gpt_6_1_terra=off","arm":"kb_clean","run":"pilot_layout_20261115","status":"pilot","scope":"means over this setting's successful calls of this form in the layout pilot (every language it ran, repeats included)","calls":500,"input_tokens_mean":336,"output_tokens_mean":12,"reasoning_tokens_mean":0,"cost_per_call":{"value":0.0004,"currency":"USD"},"latency_s":1.6}],"zh":{"run_pilot":"pilot_zh_graded_20261116","run_followup":"pilot_zh_followup_20261116","status":"pilot","classes":18,"m":2,"m_source":"by construction (design document section 2, K2): two word-alone items share each message","bound_pct":50.0,"forced":[{"id":"word_alone_one_call","label":"word alone, one call per item","k":366,"n":756,"pct":48.412698,"ci":[46.5608,50.1323],"ci_method":"boot_class","run":"pilot_zh_graded_20261116"},{"id":"word_alone_three_calls","label":"word alone, three calls per item","k":1108,"n":2268,"pct":48.853616,"ci":[46.737213,50.573192],"ci_method":"boot_class","run":"pilot_zh_followup_20261116"},{"id":"sentence","label":"other words, in a sentence","k":749,"n":756,"pct":99.074074,"ci":[98.148,99.868],"ci_method":"boot_class","run":"pilot_zh_graded_20261116","note":"Sentence items share none of the 18 classes of the word-alone items."},{"id":"sentence_rarer_word","label":"in a sentence, the rarer word is the answer","k":371,"n":378,"pct":98.148148,"ci":[96.296,99.471],"ci_method":"boot_class","run":"pilot_zh_graded_20261116"}],"flag":[{"format":"open","format_label":"free text, no closing line","outcome":"says so","sentence":"none","sentence_label":"no sentence","k":1194,"n":2268,"pct":52.645503,"ci":[44.796076,59.611993],"ci_method":"boot_class","determined":{"k":1,"n":1260,"pct":0.079365},"bare":{"k":32,"n":2268,"pct":1.410935}},{"format":"open","format_label":"free text, no closing line","outcome":"says so","sentence":"control","sentence_label":"control sentence","k":1268,"n":2268,"pct":55.908289,"ci":[47.751323,63.492063],"ci_method":"boot_class","determined":{"k":0,"n":1260,"pct":0.0},"bare":{"k":89,"n":2268,"pct":3.924162}},{"format":"open","format_label":"free text, no closing line","outcome":"says so","sentence":"note","sentence_label":"note","k":1561,"n":2268,"pct":68.82716,"ci":[60.670194,75.617284],"ci_method":"boot_class","determined":{"k":6,"n":1260,"pct":0.47619},"bare":{"k":17,"n":2268,"pct":0.749559}},{"format":"free","format_label":"free text, closing line asks for the option number","outcome":"says so","sentence":"none","sentence_label":"no sentence","k":491,"n":2268,"pct":21.64903,"ci":[18.209877,25.0],"ci_method":"boot_class","determined":{"k":3,"n":1260,"pct":0.238095},"bare":{"k":1374,"n":2268,"pct":60.582011}},{"format":"free","format_label":"free text, closing line asks for the option number","outcome":"says so","sentence":"control","sentence_label":"control sentence","k":648,"n":2268,"pct":28.571429,"ci":[23.941799,33.24515],"ci_method":"boot_class","determined":{"k":0,"n":1260,"pct":0.0},"bare":{"k":1228,"n":2268,"pct":54.144621}},{"format":"free","format_label":"free text, closing line asks for the option number","outcome":"says so","sentence":"note","sentence_label":"note","k":758,"n":2268,"pct":33.421517,"ci":[29.012346,37.87478],"ci_method":"boot_class","determined":{"k":3,"n":1260,"pct":0.238095},"bare":{"k":1145,"n":2268,"pct":50.485009}},{"format":"ab","format_label":"answer 0 offered","outcome":"answers 0","sentence":"none","sentence_label":"no sentence","k":1353,"n":2268,"pct":59.656085,"ci":[53.08642,65.740741],"ci_method":"boot_class","determined":{"k":10,"n":1260,"pct":0.793651}},{"format":"ab","format_label":"answer 0 offered","outcome":"answers 0","sentence":"control","sentence_label":"control sentence","k":1464,"n":2268,"pct":64.550265,"ci":[57.760141,70.811287],"ci_method":"boot_class","determined":{"k":33,"n":1260,"pct":2.619048}},{"format":"ab","format_label":"answer 0 offered","outcome":"answers 0","sentence":"note","sentence_label":"note","k":1619,"n":2268,"pct":71.38448,"ci":[64.858907,77.292769],"ci_method":"boot_class","determined":{"k":36,"n":1260,"pct":2.857143}}],"sentences":{"note":"Note: the Chinese text below was typed as Hanyu Pinyin without tone marks.","control":"Note: the Chinese text below was typed in lower-case letters with spaces between words."},"contrasts":[{"id":"H1","a":"open","b":"free","outcome":"says","est":30.996473,"ci":[26.234568,35.670194],"ci_settings":[18.606702,44.57672],"p_holm":3.0517578125e-05,"supported":true,"classes_pos":18,"classes_neg":0,"settings_pos":20,"settings_neg":1,"family":"fixed in advance (follow-up plan)","unit":"points"},{"id":"H2a","a":"ab_n","b":"ab_p","outcome":"zero","est":6.834215,"ci":[5.070547,8.863536],"ci_settings":[2.601411,11.948854],"p_holm":3.0517578125e-05,"supported":true,"classes_pos":18,"classes_neg":0,"settings_pos":12,"settings_neg":2,"family":"fixed in advance (follow-up plan)","unit":"points"},{"id":"H2b","a":"free_n","b":"free_p","outcome":"says","est":4.850088,"ci":[2.777778,7.186949],"ci_settings":[1.102293,9.78836],"p_holm":0.0002593994140625,"supported":true,"classes_pos":15,"classes_neg":2,"settings_pos":12,"settings_neg":2,"family":"fixed in advance (follow-up plan)","unit":"points"},{"id":"H2c","a":"open_n","b":"open_p","outcome":"says","est":12.918871,"ci":[10.757275,14.991182],"ci_settings":[7.848325,18.078704],"p_holm":3.0517578125e-05,"supported":true,"classes_pos":18,"classes_neg":0,"settings_pos":19,"settings_neg":1,"family":"fixed in advance (follow-up plan)","unit":"points"},{"id":"ab_n minus ab","a":"ab_n","b":"ab","outcome":"zero","est":11.728395,"ci":[9.259259,14.417989],"ci_settings":[6.128748,18.299162],"p":7.62939453125e-06,"supported":null,"classes_pos":18,"classes_neg":0,"family":"secondary (no test counts)","unit":"points"},{"id":"ab_p minus ab","a":"ab_p","b":"ab","outcome":"zero","est":4.89418,"ci":[2.821869,6.922399],"ci_settings":[1.322751,9.303351],"p":0.0005035400390625,"supported":null,"classes_pos":15,"classes_neg":2,"family":"secondary (no test counts)","unit":"points"},{"id":"free_n minus free","a":"free_n","b":"free","outcome":"says","est":11.772487,"ci":[9.52381,14.065256],"ci_settings":[5.820106,18.738977],"p":7.62939453125e-06,"supported":null,"classes_pos":18,"classes_neg":0,"family":"secondary (no test counts)","unit":"points"},{"id":"free_p minus free","a":"free_p","b":"free","outcome":"says","est":6.922399,"ci":[4.320988,9.567901],"ci_settings":[2.469136,12.037037],"p":0.00018310546875,"supported":null,"classes_pos":15,"classes_neg":2,"family":"secondary (no test counts)","unit":"points"},{"id":"open_n minus open","a":"open_n","b":"open","outcome":"says","est":16.181658,"ci":[12.87478,19.268078],"ci_settings":[8.597884,23.941799],"p":7.62939453125e-06,"supported":null,"classes_pos":18,"classes_neg":0,"family":"secondary (no test counts)","unit":"points"},{"id":"open_p minus open","a":"open_p","b":"open","outcome":"says","est":3.262787,"ci":[0.176367,6.17284],"ci_settings":[-3.08642,9.611993],"p":0.06219482421875,"supported":null,"classes_pos":12,"classes_neg":5,"family":"secondary (no test counts)","unit":"points"}],"per_setting":[{"setting":"claude_haiku_5_5=off","pilot_name":"haiku55_off","forced":{"k":54,"n":108,"pct":50.0,"ci":[50.0,50.0]},"arms":{"fc":{"k":54,"n":108,"bare":0,"declines":0,"ci":[0.407333,0.592667],"ci_method":"wilson_calls"},"ab":{"k":0,"n":108,"bare":0,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"ab_n":{"k":0,"n":108,"bare":0,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"ab_p":{"k":0,"n":108,"bare":0,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free":{"k":8,"n":108,"bare":28,"declines":0,"ci":[0.03801,0.139398],"ci_method":"wilson_calls"},"free_n":{"k":36,"n":108,"bare":0,"declines":0,"ci":[0.251504,0.426613],"ci_method":"wilson_calls"},"free_p":{"k":6,"n":108,"bare":1,"declines":0,"ci":[0.025707,0.115936],"ci_method":"wilson_calls"},"open":{"k":25,"n":108,"bare":0,"declines":0,"ci":[0.161993,0.319417],"ci_method":"wilson_calls"},"open_n":{"k":40,"n":108,"bare":0,"declines":0,"ci":[0.285214,0.464432],"ci_method":"wilson_calls"},"open_p":{"k":24,"n":108,"bare":0,"declines":0,"ci":[0.154124,0.309403],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":15.740741,"ci":[2.777778,30.555556]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":27.777778,"ci":[12.962963,43.518519]},"H2c":{"est":14.814815,"ci":[4.62963,25.0]}},"reasoning_share":0.0},{"setting":"claude_haiku_5_5=high","pilot_name":"haiku55_high","forced":{"k":56,"n":108,"pct":51.851852,"ci":[48.148148,56.481481]},"arms":{"fc":{"k":56,"n":108,"bare":0,"declines":0,"ci":[0.425277,0.610488],"ci_method":"wilson_calls"},"ab":{"k":39,"n":108,"bare":0,"declines":0,"ci":[0.276734,0.455029],"ci_method":"wilson_calls"},"ab_n":{"k":59,"n":108,"bare":0,"declines":0,"ci":[0.452424,0.636989],"ci_method":"wilson_calls"},"ab_p":{"k":61,"n":108,"bare":0,"declines":0,"ci":[0.470677,0.6545],"ci_method":"wilson_calls"},"free":{"k":58,"n":108,"bare":4,"declines":0,"ci":[0.443344,0.628186],"ci_method":"wilson_calls"},"free_n":{"k":64,"n":108,"bare":3,"declines":0,"ci":[0.498293,0.680532],"ci_method":"wilson_calls"},"free_p":{"k":60,"n":108,"bare":2,"declines":0,"ci":[0.461534,0.64576],"ci_method":"wilson_calls"},"open":{"k":71,"n":108,"bare":0,"declines":0,"ci":[0.56388,0.740122],"ci_method":"wilson_calls"},"open_n":{"k":83,"n":108,"bare":0,"declines":0,"ci":[0.680583,0.838007],"ci_method":"wilson_calls"},"open_p":{"k":71,"n":108,"bare":0,"declines":0,"ci":[0.56388,0.740122],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":12.037037,"ci":[0.925926,24.074074]},"H2a":{"est":-1.851852,"ci":[-17.592593,12.962963]},"H2b":{"est":3.703704,"ci":[-7.407407,14.814815]},"H2c":{"est":11.111111,"ci":[3.703704,22.222222]}},"reasoning_share":0.997619},{"setting":"claude_sonnet_5_5=off","pilot_name":"sonnet55_off","forced":{"k":52,"n":108,"pct":48.148148,"ci":[41.666667,52.777778]},"arms":{"fc":{"k":52,"n":108,"bare":0,"declines":0,"ci":[0.389512,0.574723],"ci_method":"wilson_calls"},"ab":{"k":102,"n":108,"bare":0,"declines":0,"ci":[0.884064,0.974293],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free":{"k":37,"n":108,"bare":0,"declines":0,"ci":[0.259878,0.43612],"ci_method":"wilson_calls"},"free_n":{"k":75,"n":108,"bare":0,"declines":0,"ci":[0.602132,0.773399],"ci_method":"wilson_calls"},"free_p":{"k":59,"n":108,"bare":0,"declines":0,"ci":[0.452424,0.636989],"ci_method":"wilson_calls"},"open":{"k":59,"n":108,"bare":0,"declines":0,"ci":[0.452424,0.636989],"ci_method":"wilson_calls"},"open_n":{"k":90,"n":108,"bare":0,"declines":0,"ci":[0.751871,0.891896],"ci_method":"wilson_calls"},"open_p":{"k":74,"n":108,"bare":0,"declines":0,"ci":[0.592513,0.765136],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":20.37037,"ci":[9.259259,29.652778]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":14.814815,"ci":[2.777778,25.0]},"H2c":{"est":14.814815,"ci":[5.555556,25.0]}},"reasoning_share":0.0},{"setting":"claude_sonnet_5_5=high","pilot_name":"sonnet55_high","forced":{"k":51,"n":108,"pct":47.222222,"ci":[44.444444,50.0]},"arms":{"fc":{"k":51,"n":108,"bare":0,"declines":0,"ci":[0.380648,0.565705],"ci_method":"wilson_calls"},"ab":{"k":106,"n":108,"bare":0,"declines":0,"ci":[0.93498,0.994907],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":107,"n":108,"bare":0,"declines":0,"ci":[0.949405,0.998364],"ci_method":"wilson_calls"},"free":{"k":104,"n":108,"bare":0,"declines":0,"ci":[0.908617,0.985505],"ci_method":"wilson_calls"},"free_n":{"k":105,"n":108,"bare":0,"declines":0,"ci":[0.921495,0.990509],"ci_method":"wilson_calls"},"free_p":{"k":106,"n":108,"bare":0,"declines":0,"ci":[0.93498,0.994907],"ci_method":"wilson_calls"},"open":{"k":105,"n":108,"bare":0,"declines":0,"ci":[0.921495,0.990509],"ci_method":"wilson_calls"},"open_n":{"k":103,"n":108,"bare":0,"declines":0,"ci":[0.896174,0.980065],"ci_method":"wilson_calls"},"open_p":{"k":99,"n":108,"bare":0,"declines":0,"ci":[0.849169,0.95554],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":0.925926,"ci":[-2.777778,5.555556]},"H2a":{"est":0.925926,"ci":[0.0,2.777778]},"H2b":{"est":-0.925926,"ci":[-3.703704,1.851852]},"H2c":{"est":3.703704,"ci":[0.925926,7.407407]}},"reasoning_share":0.75119},{"setting":"claude_opus_5_5=low","pilot_name":"opus55_low","forced":{"k":54,"n":108,"pct":50.0,"ci":[46.296296,54.62963]},"arms":{"fc":{"k":54,"n":108,"bare":0,"declines":0,"ci":[0.407333,0.592667],"ci_method":"wilson_calls"},"ab":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free":{"k":103,"n":108,"bare":0,"declines":0,"ci":[0.896174,0.980065],"ci_method":"wilson_calls"},"free_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"open":{"k":105,"n":108,"bare":0,"declines":0,"ci":[0.921495,0.990509],"ci_method":"wilson_calls"},"open_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"open_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":1.851852,"ci":[0.0,5.555556]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":0.0,"ci":[0.0,0.0]}},"reasoning_share":0.911905},{"setting":"gpt_6_luna=off","pilot_name":"luna_none","forced":{"k":54,"n":108,"pct":50.0,"ci":[47.222222,52.777778]},"arms":{"fc":{"k":54,"n":108,"bare":0,"declines":0,"ci":[0.407333,0.592667],"ci_method":"wilson_calls"},"ab":{"k":45,"n":108,"bare":0,"declines":0,"ci":[0.328113,0.510945],"ci_method":"wilson_calls"},"ab_n":{"k":79,"n":108,"bare":0,"declines":0,"ci":[0.641009,0.806052],"ci_method":"wilson_calls"},"ab_p":{"k":49,"n":108,"bare":0,"declines":0,"ci":[0.363011,0.547576],"ci_method":"wilson_calls"},"free":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"open":{"k":32,"n":108,"bare":0,"declines":0,"ci":[0.218377,0.38821],"ci_method":"wilson_calls"},"open_n":{"k":26,"n":108,"bare":0,"declines":0,"ci":[0.169911,0.329381],"ci_method":"wilson_calls"},"open_p":{"k":4,"n":108,"bare":24,"declines":0,"ci":[0.014495,0.091383],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":29.62963,"ci":[15.740741,45.37037]},"H2a":{"est":27.777778,"ci":[14.814815,42.592593]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":20.37037,"ci":[8.333333,32.407407]}},"reasoning_share":0.0},{"setting":"gpt_6_luna=high","pilot_name":"luna_high","forced":{"k":53,"n":108,"pct":49.074074,"ci":[44.444444,52.777778]},"arms":{"fc":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"},"ab":{"k":85,"n":108,"bare":0,"declines":0,"ci":[0.700664,0.853692],"ci_method":"wilson_calls"},"ab_n":{"k":99,"n":108,"bare":0,"declines":0,"ci":[0.849169,0.95554],"ci_method":"wilson_calls"},"ab_p":{"k":86,"n":108,"bare":0,"declines":0,"ci":[0.710785,0.861453],"ci_method":"wilson_calls"},"free":{"k":43,"n":108,"bare":63,"declines":0,"ci":[0.310855,0.492438],"ci_method":"wilson_calls"},"free_n":{"k":86,"n":108,"bare":20,"declines":0,"ci":[0.710785,0.861453],"ci_method":"wilson_calls"},"free_p":{"k":63,"n":108,"bare":42,"declines":0,"ci":[0.489055,0.671887],"ci_method":"wilson_calls"},"open":{"k":63,"n":108,"bare":0,"declines":0,"ci":[0.489055,0.671887],"ci_method":"wilson_calls"},"open_n":{"k":102,"n":108,"bare":0,"declines":0,"ci":[0.884064,0.974293],"ci_method":"wilson_calls"},"open_p":{"k":68,"n":108,"bare":3,"declines":0,"ci":[0.535568,0.714786],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":18.518519,"ci":[5.555556,31.481481]},"H2a":{"est":12.037037,"ci":[2.777778,24.074074]},"H2b":{"est":21.296296,"ci":[9.259259,34.259259]},"H2c":{"est":31.481481,"ci":[18.518519,45.37037]}},"reasoning_share":1.0},{"setting":"gpt_6_1_terra=off","pilot_name":"terra_none","forced":{"k":52,"n":108,"pct":48.148148,"ci":[39.814815,56.481481]},"arms":{"fc":{"k":52,"n":108,"bare":0,"declines":0,"ci":[0.389512,0.574723],"ci_method":"wilson_calls"},"ab":{"k":92,"n":108,"bare":0,"declines":0,"ci":[0.772827,0.906705],"ci_method":"wilson_calls"},"ab_n":{"k":94,"n":108,"bare":0,"declines":0,"ci":[0.794109,0.921188],"ci_method":"wilson_calls"},"ab_p":{"k":97,"n":108,"bare":0,"declines":0,"ci":[0.826773,0.942171],"ci_method":"wilson_calls"},"free":{"k":1,"n":108,"bare":107,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"free_n":{"k":4,"n":108,"bare":94,"declines":0,"ci":[0.014495,0.091383],"ci_method":"wilson_calls"},"free_p":{"k":1,"n":108,"bare":104,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"open":{"k":66,"n":108,"bare":1,"declines":0,"ci":[0.516864,0.697725],"ci_method":"wilson_calls"},"open_n":{"k":71,"n":108,"bare":1,"declines":0,"ci":[0.56388,0.740122],"ci_method":"wilson_calls"},"open_p":{"k":53,"n":108,"bare":13,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":60.185185,"ci":[40.740741,78.703704]},"H2a":{"est":-2.777778,"ci":[-8.333333,0.0]},"H2b":{"est":2.777778,"ci":[0.0,7.407407]},"H2c":{"est":16.666667,"ci":[2.777778,31.481481]}},"reasoning_share":0.0},{"setting":"gpt_6_1_terra=high","pilot_name":"terra_high","forced":{"k":53,"n":108,"pct":49.074074,"ci":[43.518519,53.703704]},"arms":{"fc":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"},"ab":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free":{"k":73,"n":108,"bare":29,"declines":0,"ci":[0.582932,0.756835],"ci_method":"wilson_calls"},"free_n":{"k":96,"n":108,"bare":7,"declines":0,"ci":[0.815774,0.935288],"ci_method":"wilson_calls"},"free_p":{"k":93,"n":108,"bare":12,"declines":0,"ci":[0.783424,0.913991],"ci_method":"wilson_calls"},"open":{"k":96,"n":108,"bare":0,"declines":0,"ci":[0.815774,0.935288],"ci_method":"wilson_calls"},"open_n":{"k":107,"n":108,"bare":0,"declines":0,"ci":[0.949405,0.998364],"ci_method":"wilson_calls"},"open_p":{"k":100,"n":108,"bare":6,"declines":0,"ci":[0.860602,0.96199],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":21.296296,"ci":[10.185185,35.185185]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":2.777778,"ci":[-5.555556,12.962963]},"H2c":{"est":6.481481,"ci":[0.0,16.689815]}},"reasoning_share":0.891667},{"setting":"gpt_6_1_sol=low","pilot_name":"sol_low","forced":{"k":53,"n":108,"pct":49.074074,"ci":[41.666667,54.62963]},"arms":{"fc":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"},"ab":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free":{"k":1,"n":108,"bare":107,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"free_n":{"k":12,"n":108,"bare":94,"declines":0,"ci":[0.064712,0.184226],"ci_method":"wilson_calls"},"free_p":{"k":10,"n":108,"bare":98,"declines":0,"ci":[0.051073,0.1621],"ci_method":"wilson_calls"},"open":{"k":77,"n":108,"bare":0,"declines":0,"ci":[0.621488,0.789808],"ci_method":"wilson_calls"},"open_n":{"k":99,"n":108,"bare":0,"declines":0,"ci":[0.849169,0.95554],"ci_method":"wilson_calls"},"open_p":{"k":86,"n":108,"bare":1,"declines":0,"ci":[0.710785,0.861453],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":70.37037,"ci":[50.902778,87.037037]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":1.851852,"ci":[-1.851852,5.555556]},"H2c":{"est":12.037037,"ci":[1.851852,24.074074]}},"reasoning_share":0.644048},{"setting":"gpt_6_1_sol=high","pilot_name":"sol_high","forced":{"k":50,"n":108,"pct":46.296296,"ci":[39.814815,50.0]},"arms":{"fc":{"k":50,"n":108,"bare":0,"declines":0,"ci":[0.371814,0.556656],"ci_method":"wilson_calls"},"ab":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"free":{"k":7,"n":108,"bare":101,"declines":0,"ci":[0.031748,0.127778],"ci_method":"wilson_calls"},"free_n":{"k":34,"n":108,"bare":73,"declines":0,"ci":[0.234864,0.407487],"ci_method":"wilson_calls"},"free_p":{"k":24,"n":108,"bare":84,"declines":0,"ci":[0.154124,0.309403],"ci_method":"wilson_calls"},"open":{"k":79,"n":108,"bare":0,"declines":0,"ci":[0.641009,0.806052],"ci_method":"wilson_calls"},"open_n":{"k":100,"n":108,"bare":0,"declines":0,"ci":[0.860602,0.96199],"ci_method":"wilson_calls"},"open_p":{"k":91,"n":108,"bare":1,"declines":0,"ci":[0.762311,0.899339],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":66.666667,"ci":[48.148148,83.333333]},"H2a":{"est":0.0,"ci":[0.0,0.0]},"H2b":{"est":9.259259,"ci":[1.851852,17.592593]},"H2c":{"est":8.333333,"ci":[0.0,20.37037]}},"reasoning_share":0.9875},{"setting":"gpt_6_astra=low","pilot_name":"astra_low","forced":{"k":50,"n":108,"pct":46.296296,"ci":[39.814815,50.0]},"arms":{"fc":{"k":50,"n":108,"bare":0,"declines":0,"ci":[0.371814,0.556656],"ci_method":"wilson_calls"},"ab":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_n":{"k":108,"n":108,"bare":0,"declines":0,"ci":[0.965651,1.0],"ci_method":"wilson_calls"},"ab_p":{"k":106,"n":108,"bare":0,"declines":0,"ci":[0.93498,0.994907],"ci_method":"wilson_calls"},"free":{"k":1,"n":108,"bare":107,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"free_n":{"k":12,"n":108,"bare":93,"declines":0,"ci":[0.064712,0.184226],"ci_method":"wilson_calls"},"free_p":{"k":10,"n":108,"bare":98,"declines":0,"ci":[0.051073,0.1621],"ci_method":"wilson_calls"},"open":{"k":72,"n":108,"bare":0,"declines":0,"ci":[0.573387,0.748496],"ci_method":"wilson_calls"},"open_n":{"k":98,"n":108,"bare":2,"declines":0,"ci":[0.8379,0.948927],"ci_method":"wilson_calls"},"open_p":{"k":83,"n":108,"bare":4,"declines":0,"ci":[0.680583,0.838007],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":65.740741,"ci":[50.0,80.555556]},"H2a":{"est":1.851852,"ci":[0.0,5.555556]},"H2b":{"est":1.851852,"ci":[-7.430556,13.888889]},"H2c":{"est":13.888889,"ci":[3.703704,25.925926]}},"reasoning_share":0.621429},{"setting":"deepseek_v4_1_flash=off","pilot_name":"dsflash_off","forced":{"k":54,"n":108,"pct":50.0,"ci":[46.296296,53.703704]},"arms":{"fc":{"k":54,"n":108,"bare":0,"declines":0,"ci":[0.407333,0.592667],"ci_method":"wilson_calls"},"ab":{"k":3,"n":108,"bare":0,"declines":0,"ci":[0.009491,0.078505],"ci_method":"wilson_calls"},"ab_n":{"k":6,"n":108,"bare":0,"declines":0,"ci":[0.025707,0.115936],"ci_method":"wilson_calls"},"ab_p":{"k":6,"n":108,"bare":0,"declines":0,"ci":[0.025707,0.115936],"ci_method":"wilson_calls"},"free":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":0,"n":108,"bare":106,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"open":{"k":37,"n":108,"bare":0,"declines":0,"ci":[0.259878,0.43612],"ci_method":"wilson_calls"},"open_n":{"k":26,"n":108,"bare":0,"declines":0,"ci":[0.169911,0.329381],"ci_method":"wilson_calls"},"open_p":{"k":36,"n":108,"bare":0,"declines":0,"ci":[0.251504,0.426613],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":34.259259,"ci":[17.569444,53.703704]},"H2a":{"est":0.0,"ci":[-2.777778,2.777778]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":-9.259259,"ci":[-18.518519,-0.925926]}},"reasoning_share":0.0},{"setting":"deepseek_v4_1_flash=on","pilot_name":"dsflash_on","forced":{"k":55,"n":108,"pct":50.925926,"ci":[48.148148,53.703704]},"arms":{"fc":{"k":55,"n":108,"bare":0,"declines":0,"ci":[0.41629,0.601593],"ci_method":"wilson_calls"},"ab":{"k":72,"n":108,"bare":0,"declines":0,"ci":[0.573387,0.748496],"ci_method":"wilson_calls"},"ab_n":{"k":94,"n":108,"bare":0,"declines":0,"ci":[0.794109,0.921188],"ci_method":"wilson_calls"},"ab_p":{"k":80,"n":108,"bare":0,"declines":0,"ci":[0.650833,0.81411],"ci_method":"wilson_calls"},"free":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"open":{"k":87,"n":108,"bare":1,"declines":0,"ci":[0.720963,0.869157],"ci_method":"wilson_calls"},"open_n":{"k":96,"n":108,"bare":1,"declines":0,"ci":[0.815774,0.935288],"ci_method":"wilson_calls"},"open_p":{"k":78,"n":108,"bare":4,"declines":0,"ci":[0.631227,0.797951],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":80.555556,"ci":[67.569444,91.666667]},"H2a":{"est":12.962963,"ci":[3.703704,24.074074]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":16.666667,"ci":[7.407407,25.925926]}},"reasoning_share":1.0},{"setting":"deepseek_v4_pro=off","pilot_name":"dspro_off","forced":{"k":53,"n":108,"pct":49.074074,"ci":[47.222222,50.0]},"arms":{"fc":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"},"ab":{"k":0,"n":108,"bare":0,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"ab_n":{"k":4,"n":108,"bare":0,"declines":0,"ci":[0.014495,0.091383],"ci_method":"wilson_calls"},"ab_p":{"k":0,"n":108,"bare":0,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free":{"k":0,"n":108,"bare":107,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_n":{"k":5,"n":108,"bare":85,"declines":0,"ci":[0.019935,0.103826],"ci_method":"wilson_calls"},"free_p":{"k":1,"n":108,"bare":99,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"open":{"k":14,"n":108,"bare":0,"declines":0,"ci":[0.078812,0.205891],"ci_method":"wilson_calls"},"open_n":{"k":57,"n":108,"bare":0,"declines":0,"ci":[0.434295,0.619352],"ci_method":"wilson_calls"},"open_p":{"k":25,"n":108,"bare":0,"declines":0,"ci":[0.161993,0.319417],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":12.962963,"ci":[4.62963,23.148148]},"H2a":{"est":3.703704,"ci":[0.0,8.333333]},"H2b":{"est":3.703704,"ci":[0.0,8.333333]},"H2c":{"est":29.62963,"ci":[16.666667,42.592593]}},"reasoning_share":0.0},{"setting":"doubao_seed_2_1_lite=off","pilot_name":"doubaolite_off","forced":{"k":49,"n":108,"pct":45.37037,"ci":[38.888889,50.023148]},"arms":{"fc":{"k":49,"n":108,"bare":0,"declines":0,"ci":[0.363011,0.547576],"ci_method":"wilson_calls"},"ab":{"k":13,"n":108,"bare":0,"declines":0,"ci":[0.07171,0.19511],"ci_method":"wilson_calls"},"ab_n":{"k":43,"n":108,"bare":0,"declines":0,"ci":[0.310855,0.492438],"ci_method":"wilson_calls"},"ab_p":{"k":30,"n":108,"bare":0,"declines":0,"ci":[0.202049,0.368773],"ci_method":"wilson_calls"},"free":{"k":1,"n":108,"bare":106,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":101,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":2,"n":108,"bare":106,"declines":0,"ci":[0.005093,0.06502],"ci_method":"wilson_calls"},"open":{"k":11,"n":108,"bare":0,"declines":16,"ci":[0.057829,0.173227],"ci_method":"wilson_calls"},"open_n":{"k":55,"n":108,"bare":0,"declines":0,"ci":[0.41629,0.601593],"ci_method":"wilson_calls"},"open_p":{"k":45,"n":108,"bare":4,"declines":0,"ci":[0.328113,0.510945],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":9.259259,"ci":[3.703704,16.666667]},"H2a":{"est":12.037037,"ci":[-1.851852,25.925926]},"H2b":{"est":-1.851852,"ci":[-5.555556,0.0]},"H2c":{"est":9.259259,"ci":[0.0,17.592593]}},"reasoning_share":0.0},{"setting":"doubao_seed_2_1_lite=on","pilot_name":"doubaolite_on","forced":{"k":54,"n":108,"pct":50.0,"ci":[44.444444,55.555556]},"arms":{"fc":{"k":54,"n":108,"bare":0,"declines":0,"ci":[0.407333,0.592667],"ci_method":"wilson_calls"},"ab":{"k":64,"n":108,"bare":0,"declines":0,"ci":[0.498293,0.680532],"ci_method":"wilson_calls"},"ab_n":{"k":96,"n":108,"bare":0,"declines":0,"ci":[0.815774,0.935288],"ci_method":"wilson_calls"},"ab_p":{"k":87,"n":108,"bare":0,"declines":0,"ci":[0.720963,0.869157],"ci_method":"wilson_calls"},"free":{"k":9,"n":108,"bare":61,"declines":0,"ci":[0.04446,0.150831],"ci_method":"wilson_calls"},"free_n":{"k":49,"n":108,"bare":23,"declines":0,"ci":[0.363011,0.547576],"ci_method":"wilson_calls"},"free_p":{"k":39,"n":108,"bare":30,"declines":0,"ci":[0.276734,0.455029],"ci_method":"wilson_calls"},"open":{"k":55,"n":108,"bare":0,"declines":0,"ci":[0.41629,0.601593],"ci_method":"wilson_calls"},"open_n":{"k":75,"n":108,"bare":0,"declines":0,"ci":[0.602132,0.773399],"ci_method":"wilson_calls"},"open_p":{"k":71,"n":108,"bare":0,"declines":0,"ci":[0.56388,0.740122],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":42.592593,"ci":[28.703704,57.407407]},"H2a":{"est":8.333333,"ci":[2.777778,15.763889]},"H2b":{"est":9.259259,"ci":[-1.851852,21.296296]},"H2c":{"est":3.703704,"ci":[-8.333333,15.740741]}},"reasoning_share":1.0},{"setting":"doubao_seed_2_1_pro=off","pilot_name":"doubaopro_off","forced":{"k":55,"n":108,"pct":50.925926,"ci":[50.0,52.777778]},"arms":{"fc":{"k":55,"n":108,"bare":0,"declines":0,"ci":[0.41629,0.601593],"ci_method":"wilson_calls"},"ab":{"k":52,"n":108,"bare":0,"declines":0,"ci":[0.389512,0.574723],"ci_method":"wilson_calls"},"ab_n":{"k":67,"n":108,"bare":0,"declines":0,"ci":[0.526199,0.706272],"ci_method":"wilson_calls"},"ab_p":{"k":50,"n":108,"bare":0,"declines":0,"ci":[0.371814,0.556656],"ci_method":"wilson_calls"},"free":{"k":6,"n":108,"bare":14,"declines":30,"ci":[0.025707,0.115936],"ci_method":"wilson_calls"},"free_n":{"k":14,"n":108,"bare":14,"declines":44,"ci":[0.078812,0.205891],"ci_method":"wilson_calls"},"free_p":{"k":14,"n":108,"bare":15,"declines":42,"ci":[0.078812,0.205891],"ci_method":"wilson_calls"},"open":{"k":5,"n":108,"bare":0,"declines":72,"ci":[0.019935,0.103826],"ci_method":"wilson_calls"},"open_n":{"k":49,"n":108,"bare":0,"declines":0,"ci":[0.363011,0.547576],"ci_method":"wilson_calls"},"open_p":{"k":18,"n":108,"bare":0,"declines":3,"ci":[0.108104,0.248129],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":-0.925926,"ci":[-6.481481,4.62963]},"H2a":{"est":15.740741,"ci":[1.851852,32.407407]},"H2b":{"est":-0.0,"ci":[-12.962963,12.037037]},"H2c":{"est":28.703704,"ci":[14.814815,42.592593]}},"reasoning_share":0.0},{"setting":"kimi_k3_1_pro=off","pilot_name":"kimi_off","forced":{"k":51,"n":108,"pct":47.222222,"ci":[41.666667,50.0]},"arms":{"fc":{"k":51,"n":108,"bare":0,"declines":0,"ci":[0.380648,0.565705],"ci_method":"wilson_calls"},"ab":{"k":24,"n":108,"bare":0,"declines":0,"ci":[0.154124,0.309403],"ci_method":"wilson_calls"},"ab_n":{"k":56,"n":108,"bare":0,"declines":0,"ci":[0.425277,0.610488],"ci_method":"wilson_calls"},"ab_p":{"k":28,"n":108,"bare":0,"declines":0,"ci":[0.18589,0.349167],"ci_method":"wilson_calls"},"free":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"open":{"k":7,"n":108,"bare":30,"declines":0,"ci":[0.031748,0.127778],"ci_method":"wilson_calls"},"open_n":{"k":14,"n":108,"bare":13,"declines":0,"ci":[0.078812,0.205891],"ci_method":"wilson_calls"},"open_p":{"k":7,"n":108,"bare":29,"declines":0,"ci":[0.031748,0.127778],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":6.481481,"ci":[1.851852,12.037037]},"H2a":{"est":25.925926,"ci":[12.037037,42.592593]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":6.481481,"ci":[-0.925926,13.888889]}},"reasoning_share":0.0},{"setting":"kimi_k3_1_pro=on","pilot_name":"kimi_on","forced":{"k":52,"n":108,"pct":48.148148,"ci":[42.592593,53.703704]},"arms":{"fc":{"k":52,"n":108,"bare":0,"declines":0,"ci":[0.389512,0.574723],"ci_method":"wilson_calls"},"ab":{"k":91,"n":108,"bare":0,"declines":0,"ci":[0.762311,0.899339],"ci_method":"wilson_calls"},"ab_n":{"k":102,"n":108,"bare":0,"declines":0,"ci":[0.884064,0.974293],"ci_method":"wilson_calls"},"ab_p":{"k":94,"n":108,"bare":0,"declines":0,"ci":[0.794109,0.921188],"ci_method":"wilson_calls"},"free":{"k":1,"n":108,"bare":107,"declines":0,"ci":[0.001636,0.050595],"ci_method":"wilson_calls"},"free_n":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"free_p":{"k":0,"n":108,"bare":108,"declines":0,"ci":[0.0,0.034349],"ci_method":"wilson_calls"},"open":{"k":85,"n":108,"bare":0,"declines":0,"ci":[0.700664,0.853692],"ci_method":"wilson_calls"},"open_n":{"k":89,"n":108,"bare":0,"declines":0,"ci":[0.741503,0.884382],"ci_method":"wilson_calls"},"open_p":{"k":74,"n":108,"bare":0,"declines":0,"ci":[0.592513,0.765136],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":77.777778,"ci":[65.740741,87.962963]},"H2a":{"est":7.407407,"ci":[0.0,17.592593]},"H2b":{"est":0.0,"ci":[0.0,0.0]},"H2c":{"est":13.888889,"ci":[4.62963,24.074074]}},"reasoning_share":1.0},{"setting":"glm_5_3_flash=off-requested","pilot_name":"glmflash_offreq","forced":{"k":53,"n":108,"pct":49.074074,"ci":[41.666667,56.481481]},"arms":{"fc":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"},"ab":{"k":25,"n":108,"bare":0,"declines":0,"ci":[0.161993,0.319417],"ci_method":"wilson_calls"},"ab_n":{"k":64,"n":108,"bare":0,"declines":0,"ci":[0.498293,0.680532],"ci_method":"wilson_calls"},"ab_p":{"k":43,"n":108,"bare":0,"declines":0,"ci":[0.310855,0.492438],"ci_method":"wilson_calls"},"free":{"k":38,"n":108,"bare":2,"declines":0,"ci":[0.268289,0.445592],"ci_method":"wilson_calls"},"free_n":{"k":58,"n":108,"bare":1,"declines":0,"ci":[0.443344,0.628186],"ci_method":"wilson_calls"},"free_p":{"k":52,"n":108,"bare":0,"declines":0,"ci":[0.389512,0.574723],"ci_method":"wilson_calls"},"open":{"k":43,"n":108,"bare":0,"declines":0,"ci":[0.310855,0.492438],"ci_method":"wilson_calls"},"open_n":{"k":73,"n":108,"bare":0,"declines":0,"ci":[0.582932,0.756835],"ci_method":"wilson_calls"},"open_p":{"k":53,"n":108,"bare":0,"declines":0,"ci":[0.398407,0.58371],"ci_method":"wilson_calls"}},"contrasts":{"H1":{"est":4.62963,"ci":[-7.407407,18.518519]},"H2a":{"est":19.444444,"ci":[7.407407,32.407407]},"H2b":{"est":5.555556,"ci":[-2.777778,13.888889]},"H2c":{"est":18.518519,"ci":[9.259259,28.703704]}},"reasoning_share":0.898214}],"calls":{"pilot":36288,"followup":35280},"items":{"word_alone":36,"undetermined_calls_per_arm":2268}}}